From 921080f3d2ca71985060a2273d72257c0a35c6e4 Mon Sep 17 00:00:00 2001
From: Berkeley Martinez <Berkeley@robotie.com>
Date: Sun, 1 Nov 2015 17:20:03 -0800
Subject: [PATCH] Add challenge testing

---
 .travis.yml                                |   3 +-
 package.json                               |   9 ++-
 seed/challenges/advanced-bonfires.json     |   1 -
 seed/challenges/angularjs.json             |  35 ++++++++--
 seed/challenges/basic-bonfires.json        |   8 +--
 seed/challenges/intermediate-bonfires.json |  43 ++++++------
 seed/getChallenges.js                      |  19 +++++
 seed/index.js                              |  16 ++---
 seed/test-challenges.js                    | 108 +++++++++++++++++++++++++++++
 9 files changed, 194 insertions(+), 48 deletions(-)
 create mode 100644 seed/getChallenges.js
 create mode 100644 seed/test-challenges.js

diff --git a/.travis.yml b/.travis.yml
index b6bd11e..22cfb84 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
 language: node_js
 
 node_js:
-  - 'node'
-  - '1.6.4'
+  - '4.2.1'
 
 sudo: false
diff --git a/package.json b/package.json
index 1c6f775..7879ae7 100644
--- a/package.json
+++ b/package.json
@@ -13,7 +13,11 @@
     "prestart-production": "bower cache clean && bower install && gulp build",
     "start-production": "node pm2Start",
     "lint": "eslint --ext=.js,.jsx .",
-    "test": "gulp test-challenges"
+    "lint-challenges": "jsonlint -q seed/challenges/*.json",
+    "lint-nonprofits": "jsonlint -q seed/nonprofits.json",
+    "test-challenges": "babel-node seed/test-challenges.js | tnyan",
+    "pretest": "npm run lint-challenges && npm run lint-nonprofits",
+    "test": "npm run test-challenges"
   },
   "license": "(BSD-3-Clause AND CC-BY-SA-4.0)",
   "dependencies": {
@@ -124,11 +128,14 @@
     "chai": "~1.10.0",
     "envify": "^3.4.0",
     "istanbul": "^0.3.15",
+    "jsonlint": "^1.6.2",
     "loopback-explorer": "^1.7.2",
     "loopback-testing": "^1.1.0",
     "mocha": "~2.0.1",
     "multiline": "~1.0.1",
     "supertest": "~0.15.0",
+    "tap-nyan": "0.0.2",
+    "tape": "^4.2.2",
     "vinyl-source-stream": "^1.1.0"
   }
 }
diff --git a/seed/challenges/advanced-bonfires.json b/seed/challenges/advanced-bonfires.json
index 04ab7c1..115b236 100644
--- a/seed/challenges/advanced-bonfires.json
+++ b/seed/challenges/advanced-bonfires.json
@@ -157,7 +157,6 @@
         "Global Object"
       ],
       "solutions": [
-        "var VALUES = [1, 5, 10, 25, 100, 500, 1000, 2000, 10000];\n\nfunction drawer(price, cash, cid) {\n  cash = ~~(cash * 100);\n  price = ~~(price * 100);\n  var diff = cash-price;\n  cid.forEach(function(c) {\n    c[1] = ~~(c[1] * 100);\n  });\n  var totalCid = cid.reduce(function(a, c) {\n    return a + c[1];\n  }, 0);\n  if (diff > totalCid) {\n    return \"Insufficient Funds\";\n  }\n  if (diff === totalCid) {\n    return \"Closed\";\n  }\n  \n  var change = []; \n  var index = cid.length;\n  while (diff > 0 && --index > -1) {\n    var t = 0;\n    var value = VALUES[index];\n    while (diff >= value && cid[index][1] > 0) {\n      t += value;\n      cid[index][1] -= value;\n      diff -= value;\n    }\n    if (t) {\n      change.push([cid[index][0], t/100]);\n    }\n    console.log(JSON.stringify(change));\n  }\n  // Here is your change, ma'am.\n  return change;\n}\n\n// Example cash-in-drawer array:\n// [['PENNY', 1.01],\n// ['NICKEL', 2.05],\n// ['DIME', 3.10],\n// ['QUARTER', 4.25],\n// ['ONE', 90.00],\n// ['FIVE', 55.00],\n// ['TEN', 20.00],\n// ['TWENTY', 60.00],\n// ['ONE HUNDRED', 100.00]]\n\ndrawer(19.50, 20.00, [['PENNY', 1.01], ['NICKEL', 2.05], ['DIME', 3.10], ['QUARTER', 4.25], ['ONE', 90.00], ['FIVE', 55.00], ['TEN', 20.00], ['TWENTY', 60.00], ['ONE HUNDRED', 100.00]]);\n"
       ],
       "type": "bonfire",
       "challengeType": 5,
diff --git a/seed/challenges/angularjs.json b/seed/challenges/angularjs.json
index 905c120..c55221e 100644
--- a/seed/challenges/angularjs.json
+++ b/seed/challenges/angularjs.json
@@ -14,7 +14,12 @@
       ],
       "type": "waypoint",
       "challengeType": 2,
-      "tests": [],
+      "tests": [
+        "assert(true, 'No test needed');"
+      ],
+      "solutions": [
+        "/* no test needed */"
+      ],
       "nameCn": "",
       "descriptionCn": [],
       "nameFr": "",
@@ -37,7 +42,12 @@
       ],
       "type": "waypoint",
       "challengeType": 2,
-      "tests": [],
+      "tests": [
+        "assert(true, 'No test needed');"
+      ],
+      "solutions": [
+        "/* no test needed */"
+      ],
       "nameCn": "",
       "descriptionCn": [],
       "nameFr": "",
@@ -60,7 +70,12 @@
       ],
       "type": "waypoint",
       "challengeType": 2,
-      "tests": [],
+      "tests": [
+        "assert(true, 'No test needed');"
+      ],
+      "solutions": [
+        "/* no test needed */"
+      ],
       "nameCn": "",
       "descriptionCn": [],
       "nameFr": "",
@@ -82,7 +97,12 @@
       ],
       "type": "waypoint",
       "challengeType": 2,
-      "tests": [],
+      "tests": [
+        "assert(true, 'No test needed');"
+      ],
+      "solutions": [
+        "/* no test needed */"
+      ],
       "nameCn": "",
       "descriptionCn": [],
       "nameFr": "",
@@ -105,7 +125,12 @@
       ],
       "type": "waypoint",
       "challengeType": 2,
-      "tests": [],
+      "tests": [
+        "assert(true, 'No test needed');"
+      ],
+      "solutions": [
+        "/* no test needed */"
+      ],
       "nameCn": "",
       "descriptionCn": [],
       "nameFr": "",
diff --git a/seed/challenges/basic-bonfires.json b/seed/challenges/basic-bonfires.json
index 8cef5e1..505b91f 100644
--- a/seed/challenges/basic-bonfires.json
+++ b/seed/challenges/basic-bonfires.json
@@ -116,7 +116,7 @@
         "Arithmetic Operators"
       ],
       "solutions": [
-        "function factorialize(num) {\n  return num === 1 ? 1 : num * factorialize(num-1);\n}\n\nfactorialize(5);\n"
+        "function factorialize(num) {\n  return num < 1 ? 1 : num * factorialize(num-1);\n}\n\nfactorialize(5);\n"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -169,7 +169,6 @@
         "String.toLowerCase()"
       ],
       "solutions": [
-        "function palindrome(str) {\n  var a = str.toLowerCase().replace(/[^a-z]/g, '');\n  console.log(a.split('').reverse().join(''));\n  return a == a.split('').reverse().join('');\n}\n\n\n\npalindrome(\"eye\");\npalindrome(\"A man, a plan, a canal. Panama\");\n"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -417,7 +416,6 @@
         "String.slice()"
       ],
       "solutions": [
-        "function truncate(str, num) {\n  if (str.length > num) {\n    return str.substring(0, num-3) + '...';\n  }\n  return str;\n}\n\ntruncate('A-tisket a-tasket A green and yellow basket', 11);\n"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -663,9 +661,7 @@
       "MDNlinks": [
         "Array.sort()"
       ],
-      "solutions": [
-        "function where(arr, num) {\n  // Find my place in this sorted array.\n  return num;\n}\n\nwhere([40, 60], 50);\n"
-      ],
+      "solutions": [],
       "tests": [
         "assert(where([10, 20, 30, 40, 50], 35) === 3, 'message: <code>where([10, 20, 30, 40, 50], 35)</code> should return <code>3</code>.');",
         "assert(where([10, 20, 30, 40, 50], 30) === 2, 'message: <code>where([10, 20, 30, 40, 50], 30)</code> should return <code>2</code>.');",
diff --git a/seed/challenges/intermediate-bonfires.json b/seed/challenges/intermediate-bonfires.json
index 9ae783a..6d90d9f 100644
--- a/seed/challenges/intermediate-bonfires.json
+++ b/seed/challenges/intermediate-bonfires.json
@@ -31,7 +31,7 @@
         "Array.reduce()"
       ],
       "solutions": [
-        "function sumAll(arr) {\n  var sum = 0;\n  arr.sort(function(a,b) {return a-b;});\n  for (var i = arr[0]; i <= arr[1]; i++) {\n    sum += i; \n  }\n  return sum;\n}\n\nsumAll([1, 4]);\n"
+        "function sumAll(arr) {\n  var sum = 0;\n  arr.sort(function(a,b) {return a-b;});\n  for (var i = arr[0]; i <= arr[1]; i++) {\n    sum += i; \n  }\n  return sum;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -80,7 +80,7 @@
         "Array.concat()"
       ],
       "solutions": [
-        "function diff(arr1, arr2) {\n  var newArr = [];\n  var h1 = Object.create(null);\n  arr1.forEach(function(e) {\n    h1[e] = e;\n  });\n  \n  var h2 = Object.create(null);\n  arr2.forEach(function(e) {\n    h2[e] = e;\n  });\n  \n  Object.keys(h1).forEach(function(e) {\n     if (!(e in h2)) newArr.push(h1[e]);\n  });\n  Object.keys(h2).forEach(function(e) {\n     if (!(e in h1)) newArr.push(h2[e]);\n  });\n  // Same, same; but different.\n  return newArr;\n}\n\ndiff([1, 2, 3, 5], [1, 2, 3, 4, 5]);\n"
+        "function diff(arr1, arr2) {\n  var newArr = [];\n  var h1 = Object.create(null);\n  arr1.forEach(function(e) {\n    h1[e] = e;\n  });\n  \n  var h2 = Object.create(null);\n  arr2.forEach(function(e) {\n    h2[e] = e;\n  });\n  \n  Object.keys(h1).forEach(function(e) {\n     if (!(e in h2)) newArr.push(h1[e]);\n  });\n  Object.keys(h2).forEach(function(e) {\n     if (!(e in h1)) newArr.push(h2[e]);\n  });\n  // Same, same; but different.\n  return newArr;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -138,7 +138,7 @@
         "Array.join()"
       ],
       "solutions": [
-        "function convert(num) {\n  var ref = [['M', 1000], ['CM', 900], ['D', 500], ['CD', 400], ['C', 100], ['XC', 90], ['L', 50], ['XL', 40], ['X', 10], ['IX', 9], ['V', 5], ['IV', 4], ['I', 1]];\n  var res = [];\n  ref.forEach(function(p) {\n    while (num >= p[1]) {\n      res.push(p[0]);\n      num -= p[1];\n    }\n  });\n  return res.join('');\n}\n\nconvert(36);\n"
+        "function convert(num) {\n  var ref = [['M', 1000], ['CM', 900], ['D', 500], ['CD', 400], ['C', 100], ['XC', 90], ['L', 50], ['XL', 40], ['X', 10], ['IX', 9], ['V', 5], ['IV', 4], ['I', 1]];\n  var res = [];\n  ref.forEach(function(p) {\n    while (num >= p[1]) {\n      res.push(p[0]);\n      num -= p[1];\n    }\n  });\n  return res.join('');\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -181,7 +181,7 @@
         "Object.keys()"
       ],
       "solutions": [
-        "function where(collection, source) {\n  var arr = [];\n  var keys = Object.keys(source);\n  collection.forEach(function(e) {\n    if(keys.every(function(key) {return e[key] === source[key];})) {\n      arr.push(e);  \n    }\n  });\n  return arr;\n}\n\nwhere([{ first: 'Romeo', last: 'Montague' }, { first: 'Mercutio', last: null }, { first: 'Tybalt', last: 'Capulet' }], { last: 'Capulet' });\n"
+        "function where(collection, source) {\n  var arr = [];\n  var keys = Object.keys(source);\n  collection.forEach(function(e) {\n    if(keys.every(function(key) {return e[key] === source[key];})) {\n      arr.push(e);  \n    }\n  });\n  return arr;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -227,7 +227,7 @@
         "Array.join()"
       ],
       "solutions": [
-        "function replace(str, before, after) {\n  if (before.charAt(0) === before.charAt(0).toUpperCase()) {\n    after = after.charAt(0).toUpperCase() + after.substring(1);\n  } else {\n    after = after.charAt(0).toLowerCase() + after.substring(1);\n  }\n  return str.replace(before, after);\n}\n\nreplace(\"A quick brown fox jumped over the lazy dog\", \"jumped\", \"leaped\");\n"
+        "function myReplace(str, before, after) {\n  if (before.charAt(0) === before.charAt(0).toUpperCase()) {\n    after = after.charAt(0).toUpperCase() + after.substring(1);\n  } else {\n    after = after.charAt(0).toLowerCase() + after.substring(1);\n  }\n  return str.replace(before, after);\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -273,7 +273,7 @@
         "String.split()"
       ],
       "solutions": [
-        "function translate(str) {\n  if (isVowel(str.charAt(0))) return str + \"way\";\n  var front = [];\n  str = str.split('');\n  while (str.length && !isVowel(str[0])) {\n    front.push(str.shift());\n  }\n  return [].concat(str, front).join('') + 'ay';\n}\n\nfunction isVowel(c) {\n  return ['a', 'e', 'i', 'o', 'u'].indexOf(c.toLowerCase()) !== -1;\n}\n\ntranslate(\"consonant\");\n"
+        "function translate(str) {\n  if (isVowel(str.charAt(0))) return str + \"way\";\n  var front = [];\n  str = str.split('');\n  while (str.length && !isVowel(str[0])) {\n    front.push(str.shift());\n  }\n  return [].concat(str, front).join('') + 'ay';\n}\n\nfunction isVowel(c) {\n  return ['a', 'e', 'i', 'o', 'u'].indexOf(c.toLowerCase()) !== -1;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -316,7 +316,7 @@
         "String.split()"
       ],
       "solutions": [
-        "var lookup = Object.create(null);\nlookup.A = 'T';\nlookup.T = 'A';\nlookup.C = 'G';\nlookup.G = 'C';\n\nfunction pair(str) {\n return str.split('').map(function(p) {return [p, lookup[p]];});\n}\n\npair(\"GCG\");\n"
+        "var lookup = Object.create(null);\nlookup.A = 'T';\nlookup.T = 'A';\nlookup.C = 'G';\nlookup.G = 'C';\n\nfunction pair(str) {\n return str.split('').map(function(p) {return [p, lookup[p]];});\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -357,7 +357,7 @@
         "String.fromCharCode()"
       ],
       "solutions": [
-        "function fearNotLetter(str) {\n  var s = str.split('').map(function(c) {return c.charCodeAt(0);});\n  for (var i = 1; i < s.length; i++) {\n    if (s[i]-1 != s[i-1]) {\n      return String.fromCharCode(s[i]-1);\n    }\n  }\n}\n\nfearNotLetter('abce');\n"
+        "function fearNotLetter(str) {\n  var s = str.split('').map(function(c) {return c.charCodeAt(0);});\n  for (var i = 1; i < s.length; i++) {\n    if (s[i]-1 != s[i-1]) {\n      return String.fromCharCode(s[i]-1);\n    }\n  }\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -402,7 +402,7 @@
         "Boolean Objects"
       ],
       "solutions": [
-        "function boo(bool) {\n  // What is the new fad diet for ghost developers? The Boolean.\n  return typeof(bool) === \"boolean\";\n}\n\nboo(null);\n"
+        "function boo(bool) {\n  // What is the new fad diet for ghost developers? The Boolean.\n  return typeof(bool) === \"boolean\";\n}\n\nboo(null);"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -445,7 +445,7 @@
         "Array.reduce()"
       ],
       "solutions": [
-        "function unite(arr1, arr2, arr3) {\n  return [].slice.call(arguments).reduce(function(a, b) {\n    return [].concat(a, b.filter(function(e) {return a.indexOf(e) === -1;}));\n  }, []);\n}\n\nunite([1, 2, 3], [5, 2, 1, 4], [2, 1]);\n"
+        "function unite(arr1, arr2, arr3) {\n  return [].slice.call(arguments).reduce(function(a, b) {\n    return [].concat(a, b.filter(function(e) {return a.indexOf(e) === -1;}));\n  }, []);\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -489,7 +489,7 @@
         "HTML Entities"
       ],
       "solutions": [
-        "var MAP = { '&': '&amp;',\n            '<': '&lt;',\n            '>': '&gt;',\n            '\"': '&quot;',\n            \"'\": '&apos;'};\n\nfunction convert(str) {\n  return str.replace(/[&<>\"']/g, function(c) {\n    return MAP[c];\n  });\n}\n\nconvert('Dolce & Gabbana');\n"
+        "var MAP = { '&': '&amp;',\n            '<': '&lt;',\n            '>': '&gt;',\n            '\"': '&quot;',\n            \"'\": '&apos;'};\n\nfunction convert(str) {\n  return str.replace(/[&<>\"']/g, function(c) {\n    return MAP[c];\n  });\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -531,7 +531,7 @@
         "String.replace()"
       ],
       "solutions": [
-        "function spinalCase(str) {\n  // \"It's such a fine line between stupid, and clever.\"\n  // --David St. Hubbins\n  str = str.replace(/([a-z](?=[A-Z]))/g, '$1 ');\n  return str.toLowerCase().replace(/\\ |\\_/g, '-');\n}\n\nspinalCase('This Is Spinal Tap');\n"
+        "function spinalCase(str) {\n  // \"It's such a fine line between stupid, and clever.\"\n  // --David St. Hubbins\n  str = str.replace(/([a-z](?=[A-Z]))/g, '$1 ');\n  return str.toLowerCase().replace(/\\ |\\_/g, '-');\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -574,7 +574,7 @@
         "Remainder"
       ],
       "solutions": [
-        "function sumFibs(num) {\n  var a = 1; \n  var b = 1;\n  var s = 0;\n  while (a <= num) {\n    if (a % 2 !== 0) {   \n      s += a; \n    }\n    a = [b, b=b+a][0];\n  }\n  return s;\n}\n\nsumFibs(4);\n"
+        "function sumFibs(num) {\n  var a = 1; \n  var b = 1;\n  var s = 0;\n  while (a <= num) {\n    if (a % 2 !== 0) {   \n      s += a; \n    }\n    a = [b, b=b+a][0];\n  }\n  return s;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -615,7 +615,7 @@
         "Array.push()"
       ],
       "solutions": [
-        "function eratosthenesArray(n) {\n    var primes = [];\n    if (n > 2) {\n        var half = n>>1;\n        var sieve = Array(half);\n        for (var i = 1, limit = Math.sqrt(n)>>1; i <= limit; i++) {\n            if (!sieve[i]) {\n                for (var step = 2*i+1, j = (step*step)>>1; j < half; j+=step) {\n                    sieve[j] = true;\n                }\n            }\n        }\n        primes.push(2);\n        for (var p = 1; p < half; p++) {\n            if (!sieve[p]) primes.push(2*p+1);\n        }\n    }\n    return primes;\n}\n\nfunction sumPrimes(num) {\n  return eratosthenesArray(num+1).reduce(function(a,b) {return a+b;}, 0);\n}\n\nsumPrimes(10);\n"
+        "function eratosthenesArray(n) {\n    var primes = [];\n    if (n > 2) {\n        var half = n>>1;\n        var sieve = Array(half);\n        for (var i = 1, limit = Math.sqrt(n)>>1; i <= limit; i++) {\n            if (!sieve[i]) {\n                for (var step = 2*i+1, j = (step*step)>>1; j < half; j+=step) {\n                    sieve[j] = true;\n                }\n            }\n        }\n        primes.push(2);\n        for (var p = 1; p < half; p++) {\n            if (!sieve[p]) primes.push(2*p+1);\n        }\n    }\n    return primes;\n}\n\nfunction sumPrimes(num) {\n  return eratosthenesArray(num+1).reduce(function(a,b) {return a+b;}, 0);\n}\n\nsumPrimes(10);"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -657,7 +657,7 @@
         "Smallest Common Multiple"
       ],
       "solutions": [
-        "function gcd(a, b) {\n    while (b !== 0) {\n        a = [b, b = a % b][0];\n    }\n    return a;\n}\n\nfunction lcm(a, b) {\n    return (a * b) / gcd(a, b);\n}\n\nfunction smallestCommons(arr) {\n  arr.sort(function(a,b) {return a-b;});\n  var rng = [];\n  for (var i = arr[0]; i <= arr[1]; i++) {\n    rng.push(i);\n  }\n  return rng.reduce(lcm);\n}\n\n\nsmallestCommons([1,5]);\n"
+        "function gcd(a, b) {\n    while (b !== 0) {\n        a = [b, b = a % b][0];\n    }\n    return a;\n}\n\nfunction lcm(a, b) {\n    return (a * b) / gcd(a, b);\n}\n\nfunction smallestCommons(arr) {\n  arr.sort(function(a,b) {return a-b;});\n  var rng = [];\n  for (var i = arr[0]; i <= arr[1]; i++) {\n    rng.push(i);\n  }\n  return rng.reduce(lcm);\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -695,7 +695,7 @@
         "Array.filter()"
       ],
       "solutions": [
-        "function find(arr, func) {\n  var num;\n  arr.some(function(e) {\n    if (func(e)) {\n      num = e;\n      return true;\n    }\n  });\n  return num;\n}\n\nfind([1, 2, 3, 4], function(num){ return num % 2 === 0; });\n"
+        "function find(arr, func) {\n  var num;\n  arr.some(function(e) {\n    if (func(e)) {\n      num = e;\n      return true;\n    }\n  });\n  return num;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -736,7 +736,7 @@
         "Array.shift()"
       ],
       "solutions": [
-        "(function drop(arr, func) {\n  // Drop them elements.\n  while (arr.length && !func(arr[0])) {\n    arr.shift();\n  }\n  return arr;\n}\n\ndrop([1, 2, 3], function(n) {return n < 3; });\n)"
+        "function drop(arr, func) {\n  // Drop them elements.\n  while (arr.length && !func(arr[0])) {\n    arr.shift();\n  }\n  return arr;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -776,7 +776,7 @@
         "Array.isArray()"
       ],
       "solutions": [
-        "function steamroller(arr) {\n  if (!Array.isArray(arr)) {\n    return [arr];\n  }\n  var out = [];\n  arr.forEach(function(e) {\n    steamroller(e).forEach(function(v) {\n      out.push(v);\n    });\n  });\n  return out;\n}\n\nsteamroller([1, [2], [3, [[4]]]]);\n"
+        "function steamroller(arr) {\n  if (!Array.isArray(arr)) {\n    return [arr];\n  }\n  var out = [];\n  arr.forEach(function(e) {\n    steamroller(e).forEach(function(v) {\n      out.push(v);\n    });\n  });\n  return out;\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -815,7 +815,7 @@
         "String.fromCharCode()"
       ],
       "solutions": [
-        "function binaryAgent(str) {\n  return str.split(' ').map(function(s) { return parseInt(s, 2); }).map(function(b) { return String.fromCharCode(b);}).join('');\n}\n\nbinaryAgent('01000001 01110010 01100101 01101110 00100111 01110100 00100000 01100010 01101111 01101110 01100110 01101001 01110010 01100101 01110011 00100000 01100110 01110101 01101110 00100001 00111111');\n"
+        "function binaryAgent(str) {\n  return str.split(' ').map(function(s) { return parseInt(s, 2); }).map(function(b) { return String.fromCharCode(b);}).join('');\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -858,7 +858,7 @@
         "assert.strictEqual(every([{\"single\": \"double\"}, {\"single\": NaN}], \"single\"), false, 'message: <code>every([{\"single\": \"double\"}, {\"single\": NaN}], \"single\")</code> should return false');"
       ],
       "solutions": [
-        "function every(collection, pre) {\n  // Does everyone have one of these?\n  return collection.every(function(e) { return e[pre]; });\n}\n\nevery([{'user': 'Tinky-Winky', 'sex': 'male'}, {'user': 'Dipsy', 'sex': 'male'}, {'user': 'Laa-Laa', 'sex': 'female'}, {'user': 'Po', 'sex': 'female'}], 'sex');\n"
+        "function every(collection, pre) {\n  // Does everyone have one of these?\n  return collection.every(function(e) { return e[pre]; });\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
@@ -904,8 +904,7 @@
         "Arguments object"
       ],
       "solutions": [
-        "function add() {\n  if (arguments.length == 1) {\n    var a = arguments[0];\n    if (!isNumber(a)) return;\n    return function(b) {\n      if (!isNumber(b)) return;\n      return a+b;\n    };\n  }\n  if (![].slice.call(arguments).every(isNumber)) return;\n  return arguments[0] + arguments[1];\n}\n  \nfunction isNumber(obj) {\n    return toString.call(obj) == '[object Number]';\n}\n\nadd(2,3);\n",
-        "function add() {\n  var a = arguments[0];\n  if (toString.call(a) !== '[object Number]') return;  \n  if (arguments.length === 1) {\n    return function(b) {\n      if (toString.call(b) !== '[object Number]') return;\n      return a + b;\n    };\n  }\n  var b = arguments[1];\n  if (toString.call(b) !== '[object Number]') return;  \n  return a + arguments[1];\n}\n\nadd(2,3);\n"
+        "function add() {\n  var a = arguments[0];\n  if (toString.call(a) !== '[object Number]') return;  \n  if (arguments.length === 1) {\n    return function(b) {\n      if (toString.call(b) !== '[object Number]') return;\n      return a + b;\n    };\n  }\n  var b = arguments[1];\n  if (toString.call(b) !== '[object Number]') return;  \n  return a + arguments[1];\n}"
       ],
       "type": "bonfire",
       "challengeType": 5,
diff --git a/seed/getChallenges.js b/seed/getChallenges.js
new file mode 100644
index 0000000..f86a3fc
--- /dev/null
+++ b/seed/getChallenges.js
@@ -0,0 +1,19 @@
+var fs = require('fs');
+var path = require('path');
+
+
+function getFilesFor(dir) {
+  return fs.readdirSync(path.join(__dirname, '/' + dir));
+}
+
+module.exports = function getChallenges() {
+  try {
+    return getFilesFor('challenges')
+      .map(function(file) {
+        return require('./challenges/' + file);
+      });
+  } catch (e) {
+    console.log('error', e);
+    return [];
+  }
+};
diff --git a/seed/index.js b/seed/index.js
index 8300b97..c11360d 100644
--- a/seed/index.js
+++ b/seed/index.js
@@ -2,29 +2,23 @@
 require('babel/register');
 require('dotenv').load();
 
-var fs = require('fs'),
-    Rx = require('rx'),
+var Rx = require('rx'),
     _ = require('lodash'),
-    path = require('path'),
+    getChallenges = require('./getChallenges'),
     app = require('../server/server');
 
-function getFilesFor(dir) {
-  return fs.readdirSync(path.join(__dirname, '/' + dir));
-}
 
 var Challenge = app.models.Challenge;
-var challenges = getFilesFor('challenges');
 var destroy = Rx.Observable.fromNodeCallback(Challenge.destroyAll, Challenge);
 var create = Rx.Observable.fromNodeCallback(Challenge.create, Challenge);
 
 destroy()
-  .flatMap(function() { return Rx.Observable.from(challenges); })
-  .flatMap(function(file) {
-    var challengeSpec = require('./challenges/' + file);
+  .flatMap(function() { return Rx.Observable.from(getChallenges()); })
+  .flatMap(function(challengeSpec) {
     var order = challengeSpec.order;
     var block = challengeSpec.name;
     var isBeta = !!challengeSpec.isBeta;
-    console.log('parsed %s successfully', file);
+    console.log('parsed %s successfully', block);
 
     // challenge file has no challenges...
     if (challengeSpec.challenges.length === 0) {
diff --git a/seed/test-challenges.js b/seed/test-challenges.js
new file mode 100644
index 0000000..e47ed22
--- /dev/null
+++ b/seed/test-challenges.js
@@ -0,0 +1,108 @@
+/* eslint-disable no-eval, no-process-exit */
+import _ from 'lodash';
+import { Observable } from 'rx';
+import tape from 'tape';
+import getChallenges from './getChallenges';
+
+
+function createIsAssert(t, isThing) {
+  const { assert } = t;
+  return function() {
+    const args = [...arguments];
+    args[0] = isThing(args[0]);
+    assert.apply(t, args);
+  };
+}
+
+function fillAssert(t) {
+  const assert = t.assert;
+
+  assert.isArray = createIsAssert(t, _.isArray);
+  assert.isBoolean = createIsAssert(t, _.isBoolean);
+  assert.isString = createIsAssert(t, _.isString);
+  assert.isNumber = createIsAssert(t, _.isNumber);
+  assert.isUndefined = createIsAssert(t, _.isUndefined);
+
+  assert.deepEqual = t.deepEqual;
+  assert.equal = t.equal;
+  assert.strictEqual = t.equal;
+
+  assert.sameMembers = function sameMembers() {
+    const [ first, second, ...args] = arguments;
+    assert.apply(
+      t,
+      [
+        _.difference(first, second).length === 0 &&
+        _.difference(second, first).length === 0
+      ].concat(args)
+    );
+  };
+
+  assert.includeMembers = function includeMembers() {
+    const [ first, second, ...args] = arguments;
+    assert.apply(t, [_.difference(second, first).length === 0].concat(args));
+  };
+
+  assert.match = function match() {
+    const [value, regex, ...args] = arguments;
+    assert.apply(t, [regex.test(value)].concat(args));
+  };
+
+  return assert;
+}
+
+function createTest({ title, tests = [], solutions = [] }) {
+  const plan = tests.length;
+  return Observable.fromCallback(tape)(title)
+    .doOnNext(t => solutions.length ? t.plan(plan) : t.end())
+    .flatMap(t => {
+      if (solutions.length <= 0) {
+        t.comment('No solutions for ' + title);
+        return Observable.just({
+          title,
+          type: 'missing'
+        });
+      }
+
+      return Observable.just(t)
+        .map(fillAssert)
+        /* eslint-disable no-unused-vars */
+        // assert is used within the eval
+        .doOnNext(assert => {
+        /* eslint-enable no-unused-vars */
+          solutions.forEach(solution => {
+            tests.forEach(test => {
+              eval(solution + ';;' + test);
+            });
+          });
+        })
+        .map(() => ({ title }));
+    });
+}
+
+Observable.from(getChallenges())
+  .flatMap(challengeSpec => {
+    return Observable.from(challengeSpec.challenges);
+  })
+  .flatMap(challenge => {
+    return createTest(challenge);
+  })
+  .map(({ title, type }) => {
+    if (type === 'missing') {
+      return title;
+    }
+    return false;
+  })
+  .filter(title => !!title)
+  .toArray()
+  .subscribe(
+    (noSolutions) => {
+      console.log(
+        '# These challenges have no solutions\n- [ ] ' +
+          noSolutions.join('\n- [ ] ')
+      );
+    },
+    err => { throw err; },
+    () => process.exit(0)
+  );
+
