Add langdetect to the list of benchmark libraries

Author: kefniark

docs/benchmark.md

@@ -16,11 +16,12 @@
 | -------------- | --------------------------- | ------------------- | --------------------- | -------------- | ------------------ | --------- |
 | **TinyLD**     | `yarn bench:tinyld`         | 97.7311%            | 1.9247%               | 0.3441%        | 0.0966ms.          | 930KB     |
 | **TinyLD Web** | `yarn bench:tinyld-light`   | 97.4512%            | 2.1131%               | 0.4358%        | 0.0802ms.          | **110KB** |
-| **node-cld**   | `yarn bench:cld`            | 88.9148%            | 1.7489%               | 9.3363%        | 0.0477ms.          | > 10MB    |
+| **langdetect** | `yarn bench:langdetect`     | 94.879%             | 5.1093%               | 0.0117%        | 0.4631ms.          | 1.8MB     |
+| node-cld       | `yarn bench:cld`            | 88.9148%            | 1.7489%               | 9.3363%        | 0.0477ms.          | > 10MB    |
 | node-lingua    | `yarn bench:lingua`         | 86.3093%            | 0.13%                 | 13.5607%       | 1.5695ms.          | ~100MB    |
 | franc          | `yarn bench:franc`          | 68.7783%            | 26.3432%              | 4.8785%        | 0.1626ms.          | 267KB     |
-| franc-all      | `yarn bench:franc-all`      | 61.7893%            | 33.3322%              | 4.8785%        | 0.3688ms.          | 509KB     |
 | franc-min      | `yarn bench:franc-min`      | 65.5163%            | 23.5794%              | 10.9044%       | 0.0734ms.          | **119KB** |
+| franc-all      | `yarn bench:franc-all`      | 61.7893%            | 33.3322%              | 4.8785%        | 0.3688ms.          | 509KB     |
 | languagedetect | `yarn bench:languagedetect` | 61.6068%            | 12.295%               | 26.0982%       | 0.212ms.           | **240KB** |
 
 which gives us the following graph
@@ -35,11 +36,11 @@ Let's now compare those libraries per language
 
 #### Recommended
 
-- For **NodeJS**: `TinyLD` or `node-cld` (fast and accurate)
+- For **NodeJS**: `TinyLD`, `langdetect` or `node-cld` (fast and accurate)
 - For **Browser**: `TinyLD Light` or `franc-min` (small, decent accuracy, franc is less accurate but support more languages)
 
 #### Not recommended
 
 - `node-lingua` has a quite good accuracy but is just too big and slow
-- `franc-all` is the worse in term of accuracy, not a surprise because it tries to detect 400+ languages. A technical demo to put big numbers but useless for real usage, even a language like english is barely at ~45% detection rate.
+- `franc-all` is the worse in term of accuracy, not a surprise because it tries to detect 400+ languages with only 3-grams. A technical demo to put big numbers but useless for real usage, even a language like english barely reach ~45% detection rate.
 - `languagedetect` is light but just not accurate enough, really focused on indo-european languages (support kazakh but not chinese, korean or japanese). Interesting fact, it's more accurate than franc on west european languages.

docs/language.svg

@@ -46,6 +46,7 @@
     "bench:franc": "ts-node src/benchmark/franc.ts",
     "bench:franc-all": "ts-node src/benchmark/franc-all.ts",
     "bench:franc-min": "ts-node src/benchmark/franc-min.ts",
+    "bench:langdetect": "ts-node src/benchmark/langdetect.ts",
     "bench:languagedetect": "ts-node src/benchmark/languagedetect.ts",
     "bench:lingua": "cross-env TINYLD_CONFIG=normal ts-node src/benchmark/lingua.ts",
     "bench:tinyld": "cross-env TINYLD_CONFIG=normal ts-node src/benchmark/tinyld.ts",
@@ -78,6 +79,7 @@
     "franc": "^5.0.0",
     "franc-all": "^5.0.0",
     "franc-min": "^5.0.0",
+    "langdetect": "^0.2.1",
     "languagedetect": "^2.0.0",
     "lingua-node": "^0.3.0",
     "npm-run-all": "^4.1.5",

docs/overall.svg

@@ -0,0 +1,21 @@
+import { benchmark } from './bench'
+import fs from 'fs'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { detect } = require('langdetect')
+
+function langdetect(val: string): string {
+  const res = detect(val)
+  if (res && res.length > 0) {
+    const lang = res[0].lang || ''
+    if (['zh-cn', 'zh-tw'].includes(lang)) return 'zh'
+    return lang
+  }
+  return ''
+}
+
+;(async () => {
+  const res = await benchmark(langdetect)
+  if (!fs.existsSync('./data/bench')) fs.mkdirSync('./data/bench')
+  fs.writeFileSync('./data/bench/langdetect.json', JSON.stringify(res, null, 2))
+})()

package.json

@@ -10,6 +10,7 @@ function getJSON(filepath) {
   const data = {
     tinyld: getJSON('./data/bench/tinyld.json'),
     'tinyld-light': getJSON('./data/bench/tinyld-light.json'),
+    langdetect: getJSON('./data/bench/langdetect.json'),
     cld: getJSON('./data/bench/cld.json'),
     lingua: getJSON('./data/bench/lingua.json'),
     franc: getJSON('./data/bench/franc.json'),

src/benchmark/langdetect.ts

@@ -3,9 +3,9 @@ const chartistSvg = require('chartist-svg')
 module.exports = (data, langs) => {
   var graph = {
     title: 'NodeJS Language Detection - Per Language',
-    subtitle: 'Tinyld vs Cld vs Lingua vs Franc vs Languagedetect',
+    subtitle: 'Tinyld vs Langdetect vs Cld vs Lingua vs Franc',
     labels: langs.map((x) => x.toUpperCase()),
-    series: ['tinyld', 'cld', 'lingua', 'franc', 'languagedetect'].map((lib) => {
+    series: ['tinyld', 'langdetect', 'cld', 'lingua', 'franc'].map((lib) => {
       return langs.map((lang) => {
         return data[lib].languages[lang]
       })

utils/index.js

@@ -35,6 +35,7 @@ async function generateDocBenchmark() {
   const data = {
     tinyld: getJSON('./data/bench/tinyld.json'),
     'tinyld-light': getJSON('./data/bench/tinyld-light.json'),
+    langdetect: getJSON('./data/bench/langdetect.json'),
     cld: getJSON('./data/bench/cld.json'),
     lingua: getJSON('./data/bench/lingua.json'),
     franc: getJSON('./data/bench/franc.json'),
@@ -63,11 +64,12 @@ async function generateDocBenchmark() {
 | -------------- | --------------------------- | ------------------- | --------------------- | -------------- | ------------------ | --------- |
 | **TinyLD**     | \`yarn bench:tinyld\`       | ${stats('tinyld')} | 930KB     |
 | **TinyLD Web** | \`yarn bench:tinyld-light\` | ${stats('tinyld-light')} | **110KB** |
-| **node-cld**   | \`yarn bench:cld\`            | ${stats('cld')} |  > 10MB    |
+| **langdetect** | \`yarn bench:langdetect\`     | ${stats('langdetect')} |  1.8MB    |
+| node-cld       | \`yarn bench:cld\`            | ${stats('cld')} |  > 10MB    |
 | node-lingua    | \`yarn bench:lingua\`         | ${stats('lingua')} | ~100MB     |
 | franc          | \`yarn bench:franc\`          | ${stats('franc')} |  267KB     |
-| franc-all      | \`yarn bench:franc-all\`      | ${stats('franc-all')} |  509KB     |
 | franc-min      | \`yarn bench:franc-min\`      | ${stats('franc-min')} |  **119KB** |
+| franc-all      | \`yarn bench:franc-all\`      | ${stats('franc-all')} |  509KB     |
 | languagedetect | \`yarn bench:languagedetect\` | ${stats('languagedetect')} |  **240KB** |
 
 which gives us the following graph
@@ -81,13 +83,13 @@ Let's now compare those libraries per language
 
 #### Recommended
 
-- For **NodeJS**: \`TinyLD\` or \`node-cld\` (fast and accurate)
+- For **NodeJS**: \`TinyLD\`, \`langdetect\` or \`node-cld\` (fast and accurate)
 - For **Browser**: \`TinyLD Light\` or \`franc-min\` (small, decent accuracy, franc is less accurate but support more languages)
 
 #### Not recommended
 
 - \`node-lingua\` has a quite good accuracy but is just too big and slow
-- \`franc-all\` is the worse in term of accuracy, not a surprise because it tries to detect 400+ languages. A technical demo to put big numbers but useless for real usage, even a language like english barely reach ~45% detection rate.
+- \`franc-all\` is the worse in term of accuracy, not a surprise because it tries to detect 400+ languages with only 3-grams. A technical demo to put big numbers but useless for real usage, even a language like english barely reach ~45% detection rate.
 - \`languagedetect\` is light but just not accurate enough, really focused on indo-european languages (support kazakh but not chinese, korean or japanese). Interesting fact, it's more accurate than franc on west european languages.
 `
   )

utils/language.js

@@ -1679,6 +1679,13 @@ kleur@^4.0.3:
   resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.4.tgz#8c202987d7e577766d039a8cd461934c01cda04d"
   integrity sha512-8QADVssbrFjivHWQU7KkMgptGTl6WAcSdlbBPY4uNF+mWr6DGcKrvY2w4FQJoXch7+fKMjj0dRrL75vk3k23OA==
 
+langdetect@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/langdetect/-/langdetect-0.2.1.tgz#d7ca8339497a4dd65c4df935d46e344bfea86d01"
+  integrity sha1-18qDOUl6TdZcTfk11G40S/6obQE=
+  dependencies:
+    unicode-9.0.0 "0.7.0"
+
 languagedetect@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/languagedetect/-/languagedetect-2.0.0.tgz#4b8fa2b7593b2a3a02fb1100891041c53238936c"
@@ -2947,6 +2954,11 @@ underscore@^1.12.1:
   resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.1.tgz#0c1c6bd2df54b6b69f2314066d65b6cde6fcf9d1"
   integrity sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==
 
+unicode-9.0.0@0.7.0:
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/unicode-9.0.0/-/unicode-9.0.0-0.7.0.tgz#4d303caf9c6dbf2b979943c18a384e226643b937"
+  integrity sha1-TTA8r5xtvyuXmUPBijhOImZDuTc=
+
 unicode-properties@^1.2.2:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/unicode-properties/-/unicode-properties-1.3.1.tgz#cc642b6314bde2c691d65dd94cece09ed84f1282"