check against EAL and MEW lists + sanity checking

This commit is contained in:
kumavis 2017-08-04 17:36:33 -07:00
parent ef549ea4cf
commit 6543744d5f
5 changed files with 405 additions and 3514 deletions

View File

@ -4,7 +4,7 @@
"description": "Utility for detecting phishing domains targeting Ethereum users",
"main": "src/index.js",
"scripts": {
"test": "node test"
"test": "node test | tap-summary"
},
"author": "kumavis",
"license": "ISC",
@ -12,7 +12,11 @@
"fast-levenshtein": "^2.0.6"
},
"devDependencies": {
"async": "^2.5.0",
"csv-parse": "^1.2.1",
"needle": "^1.6.0",
"punycode": "^2.1.0",
"tap-summary": "^3.0.2",
"tape": "^4.8.0"
}
}

View File

@ -42,7 +42,6 @@
"eos.ac",
"uasfwallet.com",
"ziber.io",
"etherswap.org",
"multiply-ethereum.info",
"bittrex.comze.com",
"karbon.vacau.com",
@ -72,10 +71,6 @@
"myetherwallet.com.gl",
"myetherwallet.com.im",
"myetherwallet.com.ua",
"xn--mytherwallet-fvb.com",
"xn--myetherwallt-7db.com",
"xn--myetherwallt-leb.com",
"xn--myetherwallt-yeb.com",
"secure-myetherwallet.com",
"update-myetherwallet.com",
"wwwmyetherwallet.com",
@ -86,9 +81,6 @@
"myetherwallet.cam",
"myetherwallet.cc",
"myetherwallet.co",
"myetherwallét.com",
"myetherwallèt.com",
"myėtherwallet.com",
"myetherwallet.cm",
"myetherwallet.cz",
"myetherwallet.org",
@ -103,6 +95,159 @@
"ubiqcoin.org",
"metamask.com",
"ethtrade.io",
"myetcwallet.com"
"myetcwallet.com",
"account-kigo.net",
"bitcoin-wallet.net",
"blocklichan.info",
"bloclkicihan.info",
"coindash.ml",
"eos-bonus.com",
"eos-io.info",
"ether-wallet.net",
"ethereum-wallet.info",
"ethereum-wallet.net",
"ethereumchest.net",
"reservations-kigo.net",
"reservations-lodgix.com",
"secure-liverez.com",
"secure-onerooftop.com",
"settings-liverez.com",
"software-liverez.com",
"software-lodgix.com",
"unhackableetherwallets.com",
"www-myetherwallet.com",
"etherwallet.co.za",
"etherwalletchain.com",
"etherwallets.net",
"etherwallets.nl",
"my-ethwallet.com",
"myetherwallet.com.am",
"myetherwallet.com.ht",
"myetherwalletcom.com",
"xn--myetherwalle-xoc.com",
"xn--myetherwalle-44i.com",
"xn--myetherwalle-xhk.com",
"xn--myetherwallt-cfb.com",
"xn--myetherwallt-6tb.com",
"xn--myetherwallt-xub.com",
"xn--myetherwallt-ovb.com",
"xn--myetherwallt-fwb.com",
"xn--myetherwallt-5wb.com",
"xn--myetherwallt-jzi.com",
"xn--myetherwallt-2ck.com",
"xn--myetherwallt-lok.com",
"xn--myetherwallt-lsl.com",
"xn--myetherwallt-ce6f.com",
"xn--myetherwalet-mcc.com",
"xn--myetherwalet-xhf.com",
"xn--myetherwalet-lcc.com",
"xn--myetherwaet-15ba.com",
"xn--myetherwalet-whf.com",
"xn--myetherwaet-v2ea.com",
"xn--myetherwllet-59a.com",
"xn--myetherwllet-jbb.com",
"xn--myetherwllet-wbb.com",
"xn--myetherwllet-9bb.com",
"xn--myetherwllet-ncb.com",
"xn--myetherwllet-0cb.com",
"xn--myetherwllet-5nb.com",
"xn--myetherwllet-ktd.com",
"xn--myetherwllet-mre.com",
"xn--myetherwllet-76e.com",
"xn--myetherwllet-o0l.com",
"xn--myetherwllet-c45f.com",
"xn--myetherallet-ejn.com",
"xn--myethewallet-4nf.com",
"xn--myethewallet-iof.com",
"xn--myethewallet-mpf.com",
"xn--myethewallet-6bk.com",
"xn--myethewallet-i31f.com",
"xn--myethrwallet-feb.com",
"xn--myethrwallt-fbbf.com",
"xn--myethrwallet-seb.com",
"xn--myethrwallt-rbbf.com",
"xn--myethrwallet-5eb.com",
"xn--myethrwallt-3bbf.com",
"xn--myethrwallet-0tb.com",
"xn--myethrwallt-tpbf.com",
"xn--myethrwallet-rub.com",
"xn--myethrwallt-iqbf.com",
"xn--myethrwallet-ivb.com",
"xn--myethrwallt-6qbf.com",
"xn--myethrwallet-8vb.com",
"xn--myethrwallt-vrbf.com",
"xn--myethrwallet-zwb.com",
"xn--myethrwallt-ksbf.com",
"xn--myethrwallet-dzi.com",
"xn--myethrwallt-wbif.com",
"xn--myethrwallet-wck.com",
"xn--myethrwallt-skjf.com",
"xn--myethrwallet-fok.com",
"xn--myethrwallt-fvjf.com",
"xn--myethrwallet-fsl.com",
"xn--myethrwallt-fwkf.com",
"xn--myethrwallet-5d6f.com",
"xn--myethrwallt-319ef.com",
"xn--myeterwallet-ufk.com",
"xn--myeterwallet-nrl.com",
"xn--myeterwallet-von.com",
"xn--myeterwallet-jl6c.com",
"xn--myeherwallet-ooc.com",
"xn--myeherwalle-6hci.com",
"xn--myeherwallet-v4i.com",
"xn--myeherwalle-zgii.com",
"xn--myeherwallet-ohk.com",
"xn--myeherwalle-6oji.com",
"xn--mytherwallet-ceb.com",
"xn--mythrwallet-cbbc.com",
"xn--mythrwallt-c7acf.com",
"xn--mytherwallet-peb.com",
"xn--mythrwallet-obbc.com",
"xn--mythrwallt-n7acf.com",
"xn--mytherwallet-2eb.com",
"xn--mythrwallet-0bbc.com",
"xn--mythrwallt-y7acf.com",
"xn--mytherwallet-xtb.com",
"xn--mythrwallet-qpbc.com",
"xn--mythrwallt-jlbcf.com",
"xn--mytherwallet-oub.com",
"xn--mythrwallet-fqbc.com",
"xn--mythrwallt-5lbcf.com",
"xn--mythrwallet-3qbc.com",
"xn--mythrwallt-smbcf.com",
"xn--mytherwallet-5vb.com",
"xn--mythrwallet-srbc.com",
"xn--mythrwallt-fnbcf.com",
"xn--mytherwallet-wwb.com",
"xn--mythrwallet-hsbc.com",
"xn--mythrwallt-1nbcf.com",
"xn--mytherwallet-9yi.com",
"xn--mythrwallet-tbic.com",
"xn--mythrwallt-dnhcf.com",
"xn--mytherwallet-tck.com",
"xn--mythrwallet-pkjc.com",
"xn--mythrwallt-lsicf.com",
"xn--mytherwallet-cok.com",
"xn--mythrwallet-cvjc.com",
"xn--mythrwallt-c2icf.com",
"xn--mytherwallet-csl.com",
"xn--mythrwallet-cwkc.com",
"xn--mythrwallt-c0jcf.com",
"xn--mytherwallet-2d6f.com",
"xn--mythrwallet-019ec.com",
"xn--mythrwallt-yq3ecf.com",
"xn--metherwallet-qlb.com",
"xn--metherwallet-1uf.com",
"xn--metherwallet-iyi.com",
"xn--metherwallet-zhk.com",
"xn--metherwallet-3ml.com",
"xn--mytherwallet-fvb.com",
"xn--myetherwallt-7db.com",
"xn--myetherwallt-leb.com",
"xn--myetherwallt-yeb.com",
"xn--yetherwallet-vjf.com",
"xn--yetherwallet-dfk.com",
"xn--yetherwallet-1t1f.com",
"xn--yetherwallet-634f.com"
]
}

File diff suppressed because it is too large Load Diff

View File

@ -1,78 +0,0 @@
[
"etherplan.com",
"www.etherplan.com",
"www.etheralert.io",
"etheralert.io",
"www.bity.com",
"bity.com",
"www.changelly.com",
"changelly.com",
"www.coinbase.com",
"coinbase.com",
"www.contribute.status.im",
"contribute.status.im",
"www.cryptocompare.com",
"cryptocompare.com",
"dether.io",
"www.dether.io",
"www.district0x.io",
"district0x.io",
"www.easyeth.com",
"easyeth.com",
"www.ether.cards",
"ether.cards",
"www.etherbtc.io",
"etherbtc.io",
"www.etherchain.org",
"etherchain.org",
"www.etherdomain.com",
"etherdomain.com",
"www.ethereal.capital",
"ethereal.capital",
"www.ethereum.org",
"ethereum.org",
"www.ethereumdev.io",
"ethereumdev.io",
"www.ethereumdev.kr",
"ethereumdev.kr",
"www.etherid.org",
"etherid.org",
"www.etherisc.com",
"etherisc.com",
"www.ethermine.org",
"ethermine.org",
"www.ethernodes.org",
"ethernodes.org",
"www.ethernodes.org",
"ethernodes.org",
"www.ethnews.com",
"ethnews.com",
"www.ethpool.org",
"ethpool.org",
"www.everex.cash",
"everex.cash",
"www.everex.io",
"everex.io",
"www.kraken.com",
"kraken.com",
"www.ledgerwallet.com",
"ledgerwallet.com",
"www.m.famalk.net",
"m.famalk.net",
"www.myetherapi.com",
"myetherapi.com",
"www.myetheroll.com",
"myetheroll.com",
"www.myetherwallet.com",
"myetherwallet.com",
"www.myetherwallet.groovehq.com",
"myetherwallet.groovehq.com",
"www.shapeshift.io",
"shapeshift.io",
"www.webtask.io",
"webtask.io",
"www.etherecho.com",
"etherecho.com",
"www.ethercard.io",
"ethercard.io"
]

View File

@ -1,184 +1,230 @@
const fs = require("fs")
const test = require("tape")
const needle = require('needle')
const mapValues = require('async/mapValues')
const parseCsv = require("csv-parse/lib/sync")
const punycode = require('punycode')
const PhishingDetector = require("../src/detector")
const config = require("../src/config.json")
const alexaTopSites = require("./alexa.json")
const popularDapps = require("./dapps.json")
const ealWhitelist = require("./ealWhitelist.json")
const ealBlacklist = require("./ealBlacklist.json")
// extract hits from Google Analytics data from metamask.io phishing warning
// fetch from https://analytics.google.com/analytics/web/#my-reports/N6OapMZATf-zAzHjpa9Wcw/a37075177w102798190p106879314/%3F_u.dateOption%3Dlast7days%26454-table.plotKeys%3D%5B%5D%26454-table.rowStart%3D0%26454-table.rowCount%3D250/
const rawCsv = fs.readFileSync(__dirname + '/metamaskGaq.csv', 'utf8')
const metamaskGaq = parseCsv(rawCsv, {
skip_empty_lines: true,
comment: '#',
columns: true,
}).map(row => row.Source)
const detector = new PhishingDetector(config)
const metamaskGaq = loadMetamaskGaq()
let mewBlacklist, mewWhitelist
let ealBlacklist, ealWhitelist
test("basic test", (t) => {
// blacklist
testBlacklist(t, [
"metamask.com",
"wallet-ethereum.net",
"etherclassicwallet.com",
])
// whitelist
testWhitelist(t, [
"ledgerwallet.com",
"metamask.io",
"etherscan.io",
"ethereum.org",
// whitelist subdomains
"www.metamask.io",
"faucet.metamask.io",
"zero.metamask.io",
"zero-faucet.metamask.io",
"www.myetherwallet.com",
])
// fuzzy
testFuzzylist(t, [
"metmask.io",
"myetherwallet.cx",
"myetherwallet.aaa",
"myetherwallet.za",
"myetherwallet.z",
])
// do NOT detected as phishing
testAnyType(t, false, [
"example.com",
"etherid.org",
"ether.cards",
"easyeth.com",
"etherdomain.com",
"ethnews.com",
"cryptocompare.com",
"kraken.com",
"myetherwallet.groovehq.com",
"dether.io",
"ethermine.org",
"slaask.com",
"ethereumdev.io",
"ethereumdev.kr",
"etherplan.com",
"etherplay.io",
"ethtrade.org",
"ethereumpool.co",
"estream.to",
"ethereum.os.tc",
"theethereum.wiki",
"taas.fund",
"tether.to",
"ether.direct",
"themem.io",
"metajack.im",
"mestatalsl.biz",
"thregg.com",
"steem.io",
])
// do detect as phishing
testAnyType(t, true, [
"ethtrade.io",
"myetherwallèt.com",
"myetherwallet.cm",
"myethervvallet.com",
"metherwallet.com",
"mtetherwallet.com",
"my-etherwallet.com",
"my-etherwallet.in",
"myeherwallet.com",
"myetcwallet.com",
"myetehrwallet.com",
"myeterwallet.com",
"myethe.rwallet.com",
"myethereallet.com",
"myetherieumwallet.com",
"myetherswallet.com",
"myetherw.allet.com",
"myetherwal.let.com",
"myetherwalet.com",
"myetherwaliet.com",
"myetherwall.et.com",
"myetherwaller.com",
"myetherwallett.com",
"myetherwaillet.com",
"myetherwalllet.com",
"myetherweb.com.de",
"myethetwallet.com",
"myethewallet.com",
"myÄ—therwallet.com",
"myelherwallel.com",
"mvetherwallet.com",
"myethewallet.net",
"myetherwillet.com",
"myetherwallel.com",
"myeltherwallet.com",
"myelherwallet.com",
"wwwmyetherwallet.com",
"myethermwallet.com",
])
// etc...
testNoMatch(t, [
"MetaMask",
"localhost",
"bancor",
"127.0.0.1",
])
t.end()
// load MEW blacklist
mapValues({
mewBlacklist: 'https://raw.githubusercontent.com/MyEtherWallet/ethereum-lists/master/urls-darklist.json',
mewWhitelist: 'https://raw.githubusercontent.com/MyEtherWallet/ethereum-lists/master/urls-lightlist.json',
ealWhitelist: 'https://raw.githubusercontent.com/409H/EtherAddressLookup/master/whitelists/domains.json',
ealBlacklist: 'https://raw.githubusercontent.com/409H/EtherAddressLookup/master/blacklists/domains.json',
}, (url, _, cb) => loadRemoteJson(url, cb), (err, results) => {
if (err) throw err
// parse results
mewBlacklist = results.mewBlacklist.map(entry => entry.id).filter((domain) => !domain.includes('/')).map(punycode.toASCII)
mewWhitelist = results.mewWhitelist.map(entry => entry.id).filter((domain) => !domain.includes('/')).map(punycode.toASCII)
ealBlacklist = results.ealBlacklist.filter((domain) => !domain.includes('/')).map(punycode.toASCII)
ealWhitelist = results.ealWhitelist.filter((domain) => !domain.includes('/')).map(punycode.toASCII)
startTests()
})
test("alexa top sites", (t) => {
testAnyType(t, false, alexaTopSites)
t.end()
})
function loadMetamaskGaq () {
// extract hits from Google Analytics data from metamask.io phishing warning
// fetch from https://analytics.google.com/analytics/web/#my-reports/N6OapMZATf-zAzHjpa9Wcw/a37075177w102798190p106879314/%3F_u.dateOption%3Dlast7days%26454-table.plotKeys%3D%5B%5D%26454-table.rowStart%3D0%26454-table.rowCount%3D250/
const rawCsv = fs.readFileSync(__dirname + '/metamaskGaq.csv', 'utf8')
const result = parseCsv(rawCsv, {
skip_empty_lines: true,
comment: '#',
columns: true,
}).map(row => row.Source).map(punycode.toASCII)
return result
}
test("popular dapps", (t) => {
testAnyType(t, false, popularDapps)
t.end()
})
test("eal whitelist", (t) => {
testAnyType(t, false, ealWhitelist)
t.end()
})
function startTests () {
test("eal blacklist", (t) => {
testAnyType(t, true, ealBlacklist.filter((domain) => !domain.includes('/')))
t.end()
})
test("basic test", (t) => {
// make sure all metamask phishing hits are explicitly blacklisted
test("metamask gaq", (t) => {
metamaskGaq.forEach((domain) => {
const value = detector.check(domain)
// enforcing type is optional
if (value.type === 'all') {
t.comment(`"${domain}" was NOT identified as phishing`)
}
t.notEqual(value.type, 'fuzzy', `MetaMask Gaq result: "${domain}" should NOT be "fuzzy"`)
// blacklist
testBlacklist(t, [
"metamask.com",
"wallet-ethereum.net",
"etherclassicwallet.com",
])
// whitelist
testWhitelist(t, [
"ledgerwallet.com",
"metamask.io",
"etherscan.io",
"ethereum.org",
// whitelist subdomains
"www.metamask.io",
"faucet.metamask.io",
"zero.metamask.io",
"zero-faucet.metamask.io",
"www.myetherwallet.com",
])
// fuzzy
testFuzzylist(t, [
"metmask.io",
"myetherwallet.cx",
"myetherwallet.aaa",
"myetherwallet.za",
"myetherwallet.z",
])
// do NOT detected as phishing
testAnyType(t, false, [
"example.com",
"etherid.org",
"ether.cards",
"easyeth.com",
"etherdomain.com",
"ethnews.com",
"cryptocompare.com",
"kraken.com",
"myetherwallet.groovehq.com",
"dether.io",
"ethermine.org",
"slaask.com",
"ethereumdev.io",
"ethereumdev.kr",
"etherplan.com",
"etherplay.io",
"ethtrade.org",
"ethereumpool.co",
"estream.to",
"ethereum.os.tc",
"theethereum.wiki",
"taas.fund",
"tether.to",
"ether.direct",
"themem.io",
"metajack.im",
"mestatalsl.biz",
"thregg.com",
"steem.io",
])
// do detect as phishing
testAnyType(t, true, [
"ethtrade.io",
"myetherwallèt.com",
"myetherwallet.cm",
"myethervvallet.com",
"metherwallet.com",
"mtetherwallet.com",
"my-etherwallet.com",
"my-etherwallet.in",
"myeherwallet.com",
"myetcwallet.com",
"myetehrwallet.com",
"myeterwallet.com",
"myethe.rwallet.com",
"myethereallet.com",
"myetherieumwallet.com",
"myetherswallet.com",
"myetherw.allet.com",
"myetherwal.let.com",
"myetherwalet.com",
"myetherwaliet.com",
"myetherwall.et.com",
"myetherwaller.com",
"myetherwallett.com",
"myetherwaillet.com",
"myetherwalllet.com",
"myetherweb.com.de",
"myethetwallet.com",
"myethewallet.com",
"myÄ—therwallet.com",
"myelherwallel.com",
"mvetherwallet.com",
"myethewallet.net",
"myetherwillet.com",
"myetherwallel.com",
"myeltherwallet.com",
"myelherwallet.com",
"wwwmyetherwallet.com",
"myethermwallet.com",
])
// etc...
testNoMatch(t, [
"MetaMask",
"localhost",
"bancor",
"127.0.0.1",
])
t.end()
})
t.end()
})
test("alexa top sites", (t) => {
testAnyType(t, false, alexaTopSites)
t.end()
})
function testBlacklist(t, domains) {
test("popular dapps", (t) => {
testAnyType(t, false, popularDapps)
t.end()
})
test("EAL lists", (t) => {
testListIsPunycode(t, ealWhitelist)
testListIsPunycode(t, ealBlacklist)
testAnyType(t, false, ealWhitelist)
testAnyType(t, true, ealBlacklist)
t.end()
})
test("MEW lists", (t) => {
testListIsPunycode(t, mewWhitelist)
testListIsPunycode(t, mewBlacklist)
testAnyType(t, false, mewWhitelist)
testAnyType(t, true, mewBlacklist)
t.end()
})
// make sure all metamask phishing hits are explicitly blacklisted
test("metamask gaq", (t) => {
testListIsPunycode(t, metamaskGaq)
metamaskGaq.forEach((domain) => {
const value = detector.check(domain)
// enforcing type is optional
// if (value.type === 'all') {
// t.comment(`"${domain}" was NOT identified as phishing`)
// }
t.notEqual(value.type, 'fuzzy', `MetaMask Gaq result: "${domain}" should NOT be "fuzzy"`)
})
t.end()
})
test("config exclusively using punycode", (t) => {
testListIsPunycode(t, config.whitelist)
testListIsPunycode(t, config.fuzzylist)
testListIsPunycode(t, config.blacklist)
t.end()
})
test("config not repetitive", (t) => {
testListDoesntContainRepeats(t, config.whitelist)
testListDoesntContainRepeats(t, config.fuzzylist)
testListDoesntContainRepeats(t, config.blacklist)
t.end()
})
}
function testBlacklist (t, domains) {
domains.forEach((domain) => {
testDomain(t, {
domain: domain,
@ -188,7 +234,7 @@ function testBlacklist(t, domains) {
})
}
function testWhitelist(t, domains) {
function testWhitelist (t, domains) {
domains.forEach((domain) => {
testDomain(t, {
domain: domain,
@ -198,7 +244,7 @@ function testWhitelist(t, domains) {
})
}
function testFuzzylist(t, domains) {
function testFuzzylist (t, domains) {
domains.forEach((domain) => {
testDomain(t, {
domain: domain,
@ -208,7 +254,20 @@ function testFuzzylist(t, domains) {
})
}
function testNoMatch(t, domains) {
function testListIsPunycode (t, list) {
list.forEach((domain) => {
t.equals(domain, punycode.toASCII(domain), `domain "${domain}" is encoded in punycode`)
})
}
function testListDoesntContainRepeats (t, list) {
list.forEach((domain) => {
const count = list.filter(item => item === domain).length
t.ok(count === 1, `domain "${domain}" appears in list only once`)
})
}
function testNoMatch (t, domains) {
domains.forEach((domain) => {
testDomain(t, {
domain: domain,
@ -218,7 +277,7 @@ function testNoMatch(t, domains) {
})
}
function testAnyType(t, expected, domains) {
function testAnyType (t, expected, domains) {
domains.forEach((domain) => {
testDomain(t, {
domain: domain,
@ -227,16 +286,33 @@ function testAnyType(t, expected, domains) {
})
}
function testDomain(t, { domain, type, expected }) {
function testDomain (t, { domain, type, expected }) {
const value = detector.check(domain)
// log fuzzy match for debugging
if (value.type === "fuzzy") {
t.comment(`"${domain}" fuzzy matches against "${value.match}"`)
}
// if (value.type === "fuzzy") {
// t.comment(`"${domain}" fuzzy matches against "${value.match}"`)
// }
// enforcing type is optional
if (type) {
t.equal(value.type, type, `type: "${domain}" should be "${type}"`)
}
// enforcing result is required
t.equal(value.result, expected, `result: "${domain}" should be match "${expected}"`)
}
function loadRemoteJson (url, cb) {
needle.get(url, (err, res) => {
if (err) return cb(new Error(`Trouble loading list at "${url}":\n${err.stack}`))
if (res.statusCode !== 200) {
return cb(new Error(`Trouble loading list at "${url}":\n${res.body}`))
}
let _err, result
try {
result = JSON.parse(res.body)
} catch (err) {
_err = err
}
if (err) return cb(new Error(`Trouble loading list at "${url}":\n${err.stack}`))
cb(_err, result)
})
}