test - test against metamask phishing warning traffic data

package.json

@@ -12,6 +12,7 @@
     "fast-levenshtein": "^2.0.6"
   },
   "devDependencies": {
+    "csv-parse": "^1.2.1",
     "tape": "^4.8.0"
   }
 }

src/config.json

@@ -31,7 +31,10 @@
     "etherecho.com",
     "ethereum.os.tc",
     "theethereum.wiki",
-    "metajack.im"
+    "metajack.im",
+    "etherhub.io",
+    "ethereum.network",
+    "ethereum.link"
   ],
   "blacklist": [
     "wallet-ethereum.net",
@@ -75,6 +78,23 @@
     "xn--myetherwallt-yeb.com",
     "secure-myetherwallet.com",
     "update-myetherwallet.com",
+    "wwwmyetherwallet.com",
+    "myelherwallel.com",
+    "myetherwaillet.com",
+    "myetherwaliet.com",
+    "myetherwallel.com",
+    "myetherwallet.cam",
+    "myetherwallet.cc",
+    "myetherwallet.co",
+    "myetherwallét.com",
+    "myetherwallèt.com",
+    "myėtherwallet.com",
+    "myetherwallet.cm",
+    "myetherwallet.cz",
+    "myetherwallet.org",
+    "myetherwallet.tech",
+    "myetherwallet.top",
+    "myetherwallet.net",
     "etherclassicwallet.com",
     "omg-omise.co",
     "omise-go.com",
@@ -83,7 +103,6 @@
     "ubiqcoin.org",
     "metamask.com",
     "ethtrade.io",
-    "myetcwallet.com",
-    "wwwmyetherwallet.com"
+    "myetcwallet.com"
   ]
 }

test/index.js

@@ -1,10 +1,23 @@
+const fs = require("fs")
 const test = require("tape")
+const parseCsv = require("csv-parse/lib/sync")
 const PhishingDetector = require("../src/detector")
 const config = require("../src/config.json")
 const alexaTopSites = require("./alexa.json")
 const popularDapps = require("./dapps.json")
 const ealWhitelist = require("./ealWhitelist.json")
 const ealBlacklist = require("./ealBlacklist.json")
+
+// extract hits from Google Analytics data from metamask.io phishing warning
+// fetch from https://analytics.google.com/analytics/web/#my-reports/N6OapMZATf-zAzHjpa9Wcw/a37075177w102798190p106879314/%3F_u.dateOption%3Dlast7days%26454-table.plotKeys%3D%5B%5D%26454-table.rowStart%3D0%26454-table.rowCount%3D250/
+const rawCsv = fs.readFileSync(__dirname + '/metamaskGaq.csv', 'utf8')
+const metamaskGaq = parseCsv(rawCsv, {
+  skip_empty_lines: true,
+  comment: '#',
+  columns: true,
+}).map(row => row.Source)
+
+
 const detector = new PhishingDetector(config)
 
 
@@ -74,6 +87,7 @@ test("basic test", (t) => {
     "metajack.im",
     "mestatalsl.biz",
     "thregg.com",
+    "steem.io",
   ])
 
   // do detect as phishing
@@ -150,6 +164,19 @@ test("eal blacklist", (t) => {
   t.end()
 })
 
+// make sure all metamask phishing hits are explicitly blacklisted
+test("metamask gaq", (t) => {
+  metamaskGaq.forEach((domain) => {
+    const value = detector.check(domain)
+    // enforcing type is optional
+    if (value.type === 'all') {
+      t.comment(`"${domain}" was NOT identified as phishing`)
+    }
+    t.notEqual(value.type, 'fuzzy', `MetaMask Gaq result: "${domain}" should NOT be "fuzzy"`)
+  })
+  t.end()
+})
+
 
 function testBlacklist(t, domains) {
   domains.forEach((domain) => {

test/metamaskGaq.csv

@@ -0,0 +1,110 @@
+# ----------------------------------------
+# All Web Site Data
+# Phishing
+# 20170728-20170803
+# ----------------------------------------
+
+Landing Page,Source,Sessions
+/phishing.html,ethermine.org,709
+/phishing.html,ethtrade.org,71
+/phishing.html,ethereumpool.co,29
+/phishing.html,myetherwallèt.com,23
+/phishing.html,myetherwallet.cm,22
+/phishing.html,estream.to,20
+/phishing.html,ethereum.os.tc,15
+/phishing.html,ethtrade.io,13
+/phishing.html,theethereum.wiki,13
+/phishing.html,taas.fund,12
+/phishing.html,tether.to,12
+/phishing.html,ziber.io,12
+/phishing.html,steem.io,9
+/phishing.html,the.exa.website,8
+/phishing.html,theamm.org,8
+/phishing.html,(direct),6
+/phishing.html,myetherwallet.org,6
+/phishing.html,etherbtc.io,5
+/phishing.html,ethercard.io,5
+/phishing.html,karbon.vacau.com,5
+/phishing.html,myėtherwallet.com,5
+/phishing.html,stream.moe,5
+/phishing.html,etherhub.io,4
+/phishing.html,m.famtalk.net,4
+/phishing.html,steem.supply,4
+/phishing.html,dether.io,3
+/phishing.html,etcher.io,3
+/phishing.html,ethereum.link,3
+/phishing.html,ethereumdev.io,3
+/phishing.html,maas.museum,3
+/phishing.html,mestatalsl.biz,3
+/phishing.html,myetherwaillet.com,3
+/phishing.html,myetherwallet.com.cm,3
+/phishing.html,steem.cool,3
+/phishing.html,theweek.com,3
+/phishing.html,bitherium.kr,2
+/phishing.html,coin-wallet.info,2
+/phishing.html,ethcast.com,2
+/phishing.html,etherdelta.gitlhub.io,2
+/phishing.html,everus.org,2
+/phishing.html,getstream.io,2
+/phishing.html,google,2
+/phishing.html,ithemes.com,2
+/phishing.html,multiply-ethereum.info,2
+/phishing.html,myetherwallet.net,2
+/phishing.html,omise-go.com,2
+/phishing.html,tenx-tech.com,2
+/phishing.html,tether.io,2
+/phishing.html,themem.io,2
+/phishing.html,webtask.io,2
+/phishing.html,amaok.jp,1
+/phishing.html,arzamas.academy,1
+/phishing.html,bether.co,1
+/phishing.html,bittrex.comze.com,1
+/phishing.html,btctask.net,1
+/phishing.html,district0x.net,1
+/phishing.html,dotamax.net,1
+/phishing.html,eetimes.jp,1
+/phishing.html,egaas.org,1
+/phishing.html,ethalert.com,1
+/phishing.html,ether.direct,1
+/phishing.html,etherdelta.io,1
+/phishing.html,etherdice.io,1
+/phishing.html,ethereal.capital,1
+/phishing.html,ethereum.network,1
+/phishing.html,ethereumdev.kr,1
+/phishing.html,ethermine.ru,1
+/phishing.html,etherplan.com,1
+/phishing.html,etherplay.io,1
+/phishing.html,etherscan.org,1
+/phishing.html,etherswap.org,1
+/phishing.html,ethtrade.com,1
+/phishing.html,everex.cash,1
+/phishing.html,getlorem.com,1
+/phishing.html,here.com,1
+/phishing.html,hodlethereum.com,1
+/phishing.html,jet-rack.com,1
+/phishing.html,mamahd.com,1
+/phishing.html,meanjs.org,1
+/phishing.html,mediabak.com,1
+/phishing.html,megacam.me,1
+/phishing.html,met-all.org,1
+/phishing.html,meta.vn,1
+/phishing.html,metalpay.chat,1
+/phishing.html,mtgcast.com,1
+/phishing.html,myelherwallel.com,1
+/phishing.html,myetherwaliet.com,1
+/phishing.html,myetherwallel.com,1
+/phishing.html,myetherwallet.cam,1
+/phishing.html,myetherwallet.cc,1
+/phishing.html,myetherwallet.co,1
+/phishing.html,myetherwallét.com,1
+/phishing.html,myetherwallet.com.gl,1
+/phishing.html,myetherwallet.cz,1
+/phishing.html,myetherwallet.tech,1
+/phishing.html,myetherwallet.top,1
+/phishing.html,netatama.net,1
+/phishing.html,netease.im,1
+/phishing.html,netqms.com,1
+/phishing.html,steem.it,1
+/phishing.html,steem.link,1
+/phishing.html,tapas.io,1
+/phishing.html,wheremy.com,1