Skip to content

Instantly share code, notes, and snippets.

@jerinphilip
Last active January 31, 2022 11:42

Revisions

  1. jerinphilip revised this gist Jan 31, 2022. 1 changed file with 40 additions and 0 deletions.
    40 changes: 40 additions & 0 deletions generate.py
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,40 @@
    import requests
    import re
    from slimit.parser import Parser
    from slimit import ast
    import json
    from argparse import ArgumentParser

    def recurse(node):
    if isinstance(node, ast.String):
    return node.value.replace('"', '')
    if isinstance(node, ast.Number):
    return node.value
    else:
    _dict = {}
    for child in node.children():
    _dict[child.left.value] = recurse(child.right)
    return _dict


    if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--path', type=str, required=True)
    args = parser.parse_args()

    url = "https://raw.githubusercontent.com/mozilla/firefox-translations/main/extension/model/modelRegistry.js"
    content = requests.get(url).text
    rootURL = re.compile('modelRegistryRootURL = "(.*)"').search(content)
    registry = re.compile('const (modelRegistry =)').search(content)
    registryStr = content[registry.start(1):]
    parser = Parser()
    tree = parser.parse(registryStr)
    [expression_statement] = tree.children()
    assgn = expression_statement.expr
    export = {
    "modelRegistryRootURL": rootURL.group(1),
    "modelRegistry": recurse(assgn.right)
    }

    with open(args.path, "w+") as fp:
    json.dump(export, fp, indent=4, sort_keys=False)
  2. jerinphilip revised this gist Jan 31, 2022. 1 changed file with 168 additions and 170 deletions.
    338 changes: 168 additions & 170 deletions mozilla-models.json
    Original file line number Diff line number Diff line change
    @@ -1,326 +1,324 @@
    {
    "modelRegistryRootURL": "https://storage.googleapis.com/bergamot-models-sandbox/0.2.11",
    "modelRegistry" : {
    "modelRegistryRootURL": "https://storage.googleapis.com/bergamot-models-sandbox/0.2.12",
    "modelRegistry": {
    "enit": {
    "vocab": {
    "name": "vocab.enit.spm",
    "size": 814128,
    "estimatedCompressedSize": 405338,
    "expectedSha256Hash":
    "de8cbeb79e0139304bfa47e8559f2447016bf9906225a97d3df1baed4de8f3a3"
    "size": "814128",
    "estimatedCompressedSize": "405338",
    "expectedSha256Hash": "de8cbeb79e0139304bfa47e8559f2447016bf9906225a97d3df1baed4de8f3a3"
    },
    "lex": {
    "name": "lex.50.50.enit.s2t.bin",
    "size": 4489920,
    "estimatedCompressedSize": 2409986,
    "expectedSha256Hash":
    "bb1fad3b3f6a13ebce1698cf7f39ca736c4dea4525f3dab5e1a78436f07445e6"
    "size": "4489920",
    "estimatedCompressedSize": "2409986",
    "expectedSha256Hash": "bb1fad3b3f6a13ebce1698cf7f39ca736c4dea4525f3dab5e1a78436f07445e6"
    },
    "model": {
    "name": "model.enit.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13283223,
    "expectedSha256Hash":
    "a5ce3723f62ead92a0e0373b6df0ad8e3e6d22963adb1333984206e33b8b6c61"
    "size": "17140836",
    "estimatedCompressedSize": "13283223",
    "expectedSha256Hash": "a5ce3723f62ead92a0e0373b6df0ad8e3e6d22963adb1333984206e33b8b6c61"
    }
    },
    "enpt": {
    "vocab": {
    "name": "vocab.enpt.spm",
    "size": 812781,
    "estimatedCompressedSize": 406524,
    "expectedSha256Hash":
    "633a3d782c79f7d5e4b94ab96848f47c2fdf8ba82dd99efd1742b8a696bbd0cc"
    "size": "812781",
    "estimatedCompressedSize": "406524",
    "expectedSha256Hash": "633a3d782c79f7d5e4b94ab96848f47c2fdf8ba82dd99efd1742b8a696bbd0cc"
    },
    "lex": {
    "name": "lex.50.50.enpt.s2t.bin",
    "size": 4472528,
    "estimatedCompressedSize": 2411984,
    "expectedSha256Hash":
    "1e96599123d275afa37353dfe84677a4070f013494fbdc9c52a28445cc9bc38d"
    "size": "4472528",
    "estimatedCompressedSize": "2411984",
    "expectedSha256Hash": "1e96599123d275afa37353dfe84677a4070f013494fbdc9c52a28445cc9bc38d"
    },
    "model": {
    "name": "model.enpt.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13429592,
    "expectedSha256Hash":
    "d968735704c75e33c2e183b9241f14c0b2a560d01d88a2728e5c0119a4d7fb22"
    "size": "17140836",
    "estimatedCompressedSize": "13429592",
    "expectedSha256Hash": "d968735704c75e33c2e183b9241f14c0b2a560d01d88a2728e5c0119a4d7fb22"
    }
    },
    "enru": {
    "vocab": {
    "name": "vocab.enru.spm",
    "size": 937157,
    "estimatedCompressedSize": 435776,
    "expectedSha256Hash":
    "feca2d44f01b946c85faba3b15b5eb53344bec84cd14a1a4d4a82ddd774c5edd"
    "size": "937157",
    "estimatedCompressedSize": "435776",
    "expectedSha256Hash": "feca2d44f01b946c85faba3b15b5eb53344bec84cd14a1a4d4a82ddd774c5edd"
    },
    "lex": {
    "name": "lex.50.50.enru.s2t.bin",
    "size": 3049096,
    "estimatedCompressedSize": 1579779,
    "expectedSha256Hash":
    "7bd3e2c0a72286fe1f3da65c56c49a7cd77efa5f1d1a444e2a9e769480b96ff3"
    "size": "3049096",
    "estimatedCompressedSize": "1579779",
    "expectedSha256Hash": "7bd3e2c0a72286fe1f3da65c56c49a7cd77efa5f1d1a444e2a9e769480b96ff3"
    },
    "model": {
    "name": "model.enru.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 12853987,
    "expectedSha256Hash":
    "4a45186a93b8a2dd9301c66a3b3dad580b1bcfa74aadda583ca383f9fe0dea93"
    "size": "17140836",
    "estimatedCompressedSize": "12853987",
    "expectedSha256Hash": "4a45186a93b8a2dd9301c66a3b3dad580b1bcfa74aadda583ca383f9fe0dea93"
    }
    },
    "iten": {
    "vocab": {
    "name": "vocab.iten.spm",
    "size": 814151,
    "estimatedCompressedSize": 405416,
    "expectedSha256Hash":
    "22d5ce6973be5360a921103acbe984a9bfca952a1f6c55c9cb5ef7de4fd58266"
    "size": "814151",
    "estimatedCompressedSize": "405416",
    "expectedSha256Hash": "22d5ce6973be5360a921103acbe984a9bfca952a1f6c55c9cb5ef7de4fd58266"
    },
    "lex": {
    "name": "lex.50.50.iten.s2t.bin",
    "size": 5238420,
    "estimatedCompressedSize": 2860178,
    "expectedSha256Hash":
    "357d362373022b029ee9965975a133e6f36fdb0fed749202ff578365cf0111f8"
    "size": "5238420",
    "estimatedCompressedSize": "2860178",
    "expectedSha256Hash": "357d362373022b029ee9965975a133e6f36fdb0fed749202ff578365cf0111f8"
    },
    "model": {
    "name": "model.iten.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13423308,
    "expectedSha256Hash":
    "1fae546faeb9046f80b1b7e940b37b660974ce72902778181d6cd1c30b717f35"
    "size": "17140836",
    "estimatedCompressedSize": "13423308",
    "expectedSha256Hash": "1fae546faeb9046f80b1b7e940b37b660974ce72902778181d6cd1c30b717f35"
    }
    },
    "pten": {
    "vocab": {
    "name": "vocab.pten.spm",
    "size": 812889,
    "estimatedCompressedSize": 406730,
    "expectedSha256Hash":
    "8389979e3c965688b07aeb712a7e44406e5dcdb2b84087229d26fcc71448c4ed"
    "size": "812889",
    "estimatedCompressedSize": "406730",
    "expectedSha256Hash": "8389979e3c965688b07aeb712a7e44406e5dcdb2b84087229d26fcc71448c4ed"
    },
    "lex": {
    "name": "lex.50.50.pten.s2t.bin",
    "size": 5001420,
    "estimatedCompressedSize": 2733800,
    "expectedSha256Hash":
    "212ed0ae44a6f920cd6d17ca02f0a523ba6c4b0ef5078ae310c20bc4c51484c5"
    "size": "5001420",
    "estimatedCompressedSize": "2733800",
    "expectedSha256Hash": "212ed0ae44a6f920cd6d17ca02f0a523ba6c4b0ef5078ae310c20bc4c51484c5"
    },
    "model": {
    "name": "model.pten.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13584764,
    "expectedSha256Hash":
    "6c3b7af01772022a19712410c63342ba581468c2f1aac34d7488409c4043e697"
    "size": "17140836",
    "estimatedCompressedSize": "13584764",
    "expectedSha256Hash": "6c3b7af01772022a19712410c63342ba581468c2f1aac34d7488409c4043e697"
    }
    },
    "ruen": {
    "vocab": {
    "name": "vocab.ruen.spm",
    "size": 936576,
    "estimatedCompressedSize": 435801,
    "expectedSha256Hash":
    "aaf9a325c0a988c507d0312cb6ba1a02bac7a370bcd879aedee626a40bfbda78"
    "size": "936576",
    "estimatedCompressedSize": "435801",
    "expectedSha256Hash": "aaf9a325c0a988c507d0312cb6ba1a02bac7a370bcd879aedee626a40bfbda78"
    },
    "lex": {
    "name": "lex.50.50.ruen.s2t.bin",
    "size": 5090836,
    "estimatedCompressedSize": 2684919,
    "expectedSha256Hash":
    "e6667e22f5f86be4872e3768b7184727f5dd8c9f2ccfb0639baabcb1176f5d11"
    "size": "5090836",
    "estimatedCompressedSize": "2684919",
    "expectedSha256Hash": "e6667e22f5f86be4872e3768b7184727f5dd8c9f2ccfb0639baabcb1176f5d11"
    },
    "model": {
    "name": "model.ruen.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13108893,
    "expectedSha256Hash":
    "3b6a0305e3d232fadd54f5a765365b7b96ad6d8f2e818cba594b02fbd8fadb3d"
    "size": "17140836",
    "estimatedCompressedSize": "13108893",
    "expectedSha256Hash": "3b6a0305e3d232fadd54f5a765365b7b96ad6d8f2e818cba594b02fbd8fadb3d"
    }
    },
    "csen": {
    "vocab": {
    "name": "vocab.csen.spm",
    "size": 769763,
    "estimatedCompressedSize": 366392,
    "expectedSha256Hash":
    "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3"
    "size": "769763",
    "estimatedCompressedSize": "366392",
    "expectedSha256Hash": "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3"
    },
    "lex": {
    "name": "lex.50.50.csen.s2t.bin",
    "size": 4535788,
    "estimatedCompressedSize": 2418488,
    "expectedSha256Hash":
    "8228a3c3f7887759a62b7d7c674a7bef9b70161913f9b0939ab58f71186835c2"
    "size": "4535788",
    "estimatedCompressedSize": "2418488",
    "expectedSha256Hash": "8228a3c3f7887759a62b7d7c674a7bef9b70161913f9b0939ab58f71186835c2"
    },
    "model": {
    "name": "model.csen.intgemm.alphas.bin",
    "size": 17140756,
    "estimatedCompressedSize": 13045032,
    "expectedSha256Hash":
    "5b16661e2864dc50b2f4091a16bdd4ec8d8283e04271e602159ba348df5d6e2d"
    "size": "17140756",
    "estimatedCompressedSize": "13045032",
    "expectedSha256Hash": "5b16661e2864dc50b2f4091a16bdd4ec8d8283e04271e602159ba348df5d6e2d"
    }
    },
    "deen": {
    "vocab": {
    "name": "vocab.deen.spm",
    "size": 784269,
    "estimatedCompressedSize": 410738,
    "expectedSha256Hash":
    "417668f2ed297970febafb5b079a9d5ebc4ed0b3550ac8386d67a90473a09bd7"
    "size": "784269",
    "estimatedCompressedSize": "410738",
    "expectedSha256Hash": "417668f2ed297970febafb5b079a9d5ebc4ed0b3550ac8386d67a90473a09bd7"
    },
    "lex": {
    "name": "lex.50.50.deen.s2t.bin",
    "size": 5047568,
    "estimatedCompressedSize": 2657472,
    "expectedSha256Hash":
    "2f7c0f7bbce97ae5b52454074a892ba7b7610fb98e3c5d341e4ca79f0850c4de"
    "size": "5047568",
    "estimatedCompressedSize": "2657472",
    "expectedSha256Hash": "2f7c0f7bbce97ae5b52454074a892ba7b7610fb98e3c5d341e4ca79f0850c4de"
    },
    "model": {
    "name": "model.deen.intgemm.alphas.bin",
    "size": 17140837,
    "estimatedCompressedSize": 13091214,
    "expectedSha256Hash":
    "dda44d87ab0d8ad3b3871122fd3ee385f37878183a8b4ec139cd909531ec5009"
    "size": "17140837",
    "estimatedCompressedSize": "12995752",
    "expectedSha256Hash": "1980225d00dc5645491777accff5b3c9d20b92eff67a25135f1cf8fe2ed8fb8f"
    }
    },
    "encs": {
    "vocab": {
    "name": "vocab.csen.spm",
    "size": 769763,
    "estimatedCompressedSize": 366392,
    "expectedSha256Hash":
    "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3"
    "size": "769763",
    "estimatedCompressedSize": "366392",
    "expectedSha256Hash": "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3"
    },
    "lex": {
    "name": "lex.50.50.encs.s2t.bin",
    "size": 3556124,
    "estimatedCompressedSize": 1913246,
    "expectedSha256Hash":
    "e19c77231bf977988e31ff8db15fe79966b5170564bd3e10613f239e7f461d97"
    "size": "3556124",
    "estimatedCompressedSize": "1913246",
    "expectedSha256Hash": "e19c77231bf977988e31ff8db15fe79966b5170564bd3e10613f239e7f461d97"
    },
    "model": {
    "name": "model.encs.intgemm.alphas.bin",
    "size": 17140756,
    "estimatedCompressedSize": 12630325,
    "expectedSha256Hash":
    "9a2fe0588bd972accfc801e2f31c945de0557804a91666ae5ab43b94fb74ac4b"
    "size": "17140756",
    "estimatedCompressedSize": "12630325",
    "expectedSha256Hash": "9a2fe0588bd972accfc801e2f31c945de0557804a91666ae5ab43b94fb74ac4b"
    }
    },
    "ende": {
    "vocab": {
    "name": "vocab.deen.spm",
    "size": 797501,
    "estimatedCompressedSize": 412505,
    "expectedSha256Hash":
    "bc8f8229933d8294c727f3eab12f6f064e7082b929f2d29494c8a1e619ba174c"
    "size": "784269",
    "estimatedCompressedSize": "410171",
    "expectedSha256Hash": "417668f2ed297970febafb5b079a9d5ebc4ed0b3550ac8386d67a90473a09bd7"
    },
    "lex": {
    "name": "lex.50.50.ende.s2t.bin",
    "size": 3062492,
    "estimatedCompressedSize": 1575385,
    "expectedSha256Hash":
    "764797d075f0642c0b079cce6547348d65fe4e92ac69fa6a8605cd8b53dacb3f"
    "size": "3943644",
    "estimatedCompressedSize": "2113181",
    "expectedSha256Hash": "f03eb8245042feb7a5800815b8d0dc215d7a60691632405b65c461d250cedbe6"
    },
    "model": {
    "name": "model.ende.intgemm.alphas.bin",
    "size": 17140498,
    "estimatedCompressedSize": 12995752,
    "expectedSha256Hash":
    "1980225d00dc5645491777accff5b3c9d20b92eff67a25135f1cf8fe2ed8fb8f"
    "size": "17140835",
    "estimatedCompressedSize": "12768493",
    "expectedSha256Hash": "b3e980d6602ab0bdfe8d9315cb5fc282a16bb1c8dccf38e70945c584551c4318"
    }
    },
    "enes": {
    "vocab": {
    "name": "vocab.esen.spm",
    "size": 825463,
    "estimatedCompressedSize": 414566,
    "expectedSha256Hash":
    "909b1eea1face0d7f90a474fe29a8c0fef8d104b6e41e65616f864c964ba8845"
    "size": "825463",
    "estimatedCompressedSize": "414566",
    "expectedSha256Hash": "909b1eea1face0d7f90a474fe29a8c0fef8d104b6e41e65616f864c964ba8845"
    },
    "lex": {
    "name": "lex.50.50.enes.s2t.bin",
    "size": 3347104,
    "estimatedCompressedSize": 1720700,
    "expectedSha256Hash":
    "3a113d713dec3cf1d12bba5b138ae616e28bba4bbc7fe7fd39ba145e26b86d7f"
    "size": "3347104",
    "estimatedCompressedSize": "1720700",
    "expectedSha256Hash": "3a113d713dec3cf1d12bba5b138ae616e28bba4bbc7fe7fd39ba145e26b86d7f"
    },
    "model": {
    "name": "model.enes.intgemm.alphas.bin",
    "size": 17140755,
    "estimatedCompressedSize": 12602853,
    "expectedSha256Hash":
    "fa7460037a3163e03fe1d23602f964bff2331da6ee813637e092ddf37156ef53"
    "size": "17140755",
    "estimatedCompressedSize": "12602853",
    "expectedSha256Hash": "fa7460037a3163e03fe1d23602f964bff2331da6ee813637e092ddf37156ef53"
    }
    },
    "enet": {
    "vocab": {
    "name": "vocab.eten.spm",
    "size": 828426,
    "estimatedCompressedSize": 416995,
    "expectedSha256Hash":
    "e3b66bc141f6123cd40746e2fb9b8ee4f89cbf324ab27d6bbf3782e52f15fa2d"
    "size": "828426",
    "estimatedCompressedSize": "416995",
    "expectedSha256Hash": "e3b66bc141f6123cd40746e2fb9b8ee4f89cbf324ab27d6bbf3782e52f15fa2d"
    },
    "lex": {
    "name": "lex.50.50.enet.s2t.bin",
    "size": 2700780,
    "estimatedCompressedSize": 1336443,
    "expectedSha256Hash":
    "3d1b40ff43ebef82cf98d416a88a1ea19eb325a85785eef102f59878a63a829d"
    "size": "2700780",
    "estimatedCompressedSize": "1336443",
    "expectedSha256Hash": "3d1b40ff43ebef82cf98d416a88a1ea19eb325a85785eef102f59878a63a829d"
    },
    "model": {
    "name": "model.enet.intgemm.alphas.bin",
    "size": 17140754,
    "estimatedCompressedSize": 12543318,
    "expectedSha256Hash":
    "a28874a8b702a519a14dc71bcee726a5cb4b539eeaada2d06492f751469a1fd6"
    "size": "17140754",
    "estimatedCompressedSize": "12543318",
    "expectedSha256Hash": "a28874a8b702a519a14dc71bcee726a5cb4b539eeaada2d06492f751469a1fd6"
    }
    },
    "esen": {
    "vocab": {
    "name": "vocab.esen.spm",
    "size": 825463,
    "estimatedCompressedSize": 414566,
    "expectedSha256Hash":
    "909b1eea1face0d7f90a474fe29a8c0fef8d104b6e41e65616f864c964ba8845"
    "size": "825463",
    "estimatedCompressedSize": "414566",
    "expectedSha256Hash": "909b1eea1face0d7f90a474fe29a8c0fef8d104b6e41e65616f864c964ba8845"
    },
    "lex": {
    "name": "lex.50.50.esen.s2t.bin",
    "size": 3860888,
    "estimatedCompressedSize": 1978538,
    "expectedSha256Hash":
    "f11a2c23ef85ab1fee1c412b908d69bc20d66fd59faa8f7da5a5f0347eddf969"
    "size": "3860888",
    "estimatedCompressedSize": "1978538",
    "expectedSha256Hash": "f11a2c23ef85ab1fee1c412b908d69bc20d66fd59faa8f7da5a5f0347eddf969"
    },
    "model": {
    "name": "model.esen.intgemm.alphas.bin",
    "size": 17140755,
    "estimatedCompressedSize": 13215960,
    "expectedSha256Hash":
    "4b6b7f451094aaa447d012658af158ffc708fc8842dde2f871a58404f5457fe0"
    "size": "17140755",
    "estimatedCompressedSize": "13215960",
    "expectedSha256Hash": "4b6b7f451094aaa447d012658af158ffc708fc8842dde2f871a58404f5457fe0"
    }
    },
    "eten": {
    "vocab": {
    "name": "vocab.eten.spm",
    "size": 828426,
    "estimatedCompressedSize": 416995,
    "expectedSha256Hash":
    "e3b66bc141f6123cd40746e2fb9b8ee4f89cbf324ab27d6bbf3782e52f15fa2d"
    "size": "828426",
    "estimatedCompressedSize": "416995",
    "expectedSha256Hash": "e3b66bc141f6123cd40746e2fb9b8ee4f89cbf324ab27d6bbf3782e52f15fa2d"
    },
    "lex": {
    "name": "lex.50.50.eten.s2t.bin",
    "size": 3974944,
    "estimatedCompressedSize": 1920655,
    "expectedSha256Hash":
    "6992bedc590e60e610a28129c80746fe5f33144a4520e2c5508d87db14ca54f8"
    "size": "3974944",
    "estimatedCompressedSize": "1920655",
    "expectedSha256Hash": "6992bedc590e60e610a28129c80746fe5f33144a4520e2c5508d87db14ca54f8"
    },
    "model": {
    "name": "model.eten.intgemm.alphas.bin",
    "size": 17140754,
    "estimatedCompressedSize": 12222624,
    "expectedSha256Hash":
    "aac98a2371e216ee2d4843cbe896c617f6687501e17225ac83482eba52fd0028"
    "size": "17140754",
    "estimatedCompressedSize": "12222624",
    "expectedSha256Hash": "aac98a2371e216ee2d4843cbe896c617f6687501e17225ac83482eba52fd0028"
    }
    },
    "bgen": {
    "vocab": {
    "name": "vocab.bgen.spm",
    "size": "920621",
    "estimatedCompressedSize": "435213",
    "expectedSha256Hash": "24ce87ba39216714f222ca6a105f30b1863a7ef8b58c9fafdc7a66184e9813a5"
    },
    "lex": {
    "name": "lex.50.50.bgen.s2t.bin",
    "size": "6182512",
    "estimatedCompressedSize": "3272580",
    "expectedSha256Hash": "71e8d040a2f63705bec232cd186f32e9f9a78e7968216516c4535589f6a828f9"
    },
    "model": {
    "name": "model.bgen.intgemm.alphas.bin",
    "size": "17140899",
    "estimatedCompressedSize": "13167979",
    "expectedSha256Hash": "71900847a98cf66bd1d05eaafc23a794c8c1285fb3f0e2ecd2849e6f81c79d53"
    }
    },
    "enbg": {
    "vocab": {
    "name": "vocab.bgen.spm",
    "size": "919745",
    "estimatedCompressedSize": "435044",
    "expectedSha256Hash": "b14e44beb653db924c826e1696bcfab23ca9fd3e479baf8bea67d0be77432192"
    },
    "lex": {
    "name": "lex.50.50.enbg.s2t.bin",
    "size": "5607608",
    "estimatedCompressedSize": "2950444",
    "expectedSha256Hash": "0f9b794b6f8a9c4b5b781fde49391852b398184b730f89a09428cf562e8bede6"
    },
    "model": {
    "name": "model.enbg.intgemm.alphas.bin",
    "size": "17140899",
    "estimatedCompressedSize": "13311038",
    "expectedSha256Hash": "02715a7a81a610a37439d4f788a6f3efcc1ecb39618bc4184442a39378907dfe"
    }
    }
    }
  3. jerinphilip revised this gist Jan 28, 2022. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions mozilla-models.json
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,5 @@
    {
    "modelRegistryRootURL": "https://storage.googleapis.com/bergamot-models-sandbox/0.2.10",
    "modelRegistryRootURL": "https://storage.googleapis.com/bergamot-models-sandbox/0.2.11",
    "modelRegistry" : {
    "enit": {
    "vocab": {
    @@ -226,9 +226,9 @@
    "model": {
    "name": "model.ende.intgemm.alphas.bin",
    "size": 17140498,
    "estimatedCompressedSize": 13207068,
    "estimatedCompressedSize": 12995752,
    "expectedSha256Hash":
    "f0946515c6645304f0706fa66a051c3b7b7c507f12d0c850f276c18165a10c14"
    "1980225d00dc5645491777accff5b3c9d20b92eff67a25135f1cf8fe2ed8fb8f"
    }
    },
    "enes": {
  4. jerinphilip created this gist Jan 28, 2022.
    327 changes: 327 additions & 0 deletions mozilla-models.json
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,327 @@
    {
    "modelRegistryRootURL": "https://storage.googleapis.com/bergamot-models-sandbox/0.2.10",
    "modelRegistry" : {
    "enit": {
    "vocab": {
    "name": "vocab.enit.spm",
    "size": 814128,
    "estimatedCompressedSize": 405338,
    "expectedSha256Hash":
    "de8cbeb79e0139304bfa47e8559f2447016bf9906225a97d3df1baed4de8f3a3"
    },
    "lex": {
    "name": "lex.50.50.enit.s2t.bin",
    "size": 4489920,
    "estimatedCompressedSize": 2409986,
    "expectedSha256Hash":
    "bb1fad3b3f6a13ebce1698cf7f39ca736c4dea4525f3dab5e1a78436f07445e6"
    },
    "model": {
    "name": "model.enit.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13283223,
    "expectedSha256Hash":
    "a5ce3723f62ead92a0e0373b6df0ad8e3e6d22963adb1333984206e33b8b6c61"
    }
    },
    "enpt": {
    "vocab": {
    "name": "vocab.enpt.spm",
    "size": 812781,
    "estimatedCompressedSize": 406524,
    "expectedSha256Hash":
    "633a3d782c79f7d5e4b94ab96848f47c2fdf8ba82dd99efd1742b8a696bbd0cc"
    },
    "lex": {
    "name": "lex.50.50.enpt.s2t.bin",
    "size": 4472528,
    "estimatedCompressedSize": 2411984,
    "expectedSha256Hash":
    "1e96599123d275afa37353dfe84677a4070f013494fbdc9c52a28445cc9bc38d"
    },
    "model": {
    "name": "model.enpt.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13429592,
    "expectedSha256Hash":
    "d968735704c75e33c2e183b9241f14c0b2a560d01d88a2728e5c0119a4d7fb22"
    }
    },
    "enru": {
    "vocab": {
    "name": "vocab.enru.spm",
    "size": 937157,
    "estimatedCompressedSize": 435776,
    "expectedSha256Hash":
    "feca2d44f01b946c85faba3b15b5eb53344bec84cd14a1a4d4a82ddd774c5edd"
    },
    "lex": {
    "name": "lex.50.50.enru.s2t.bin",
    "size": 3049096,
    "estimatedCompressedSize": 1579779,
    "expectedSha256Hash":
    "7bd3e2c0a72286fe1f3da65c56c49a7cd77efa5f1d1a444e2a9e769480b96ff3"
    },
    "model": {
    "name": "model.enru.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 12853987,
    "expectedSha256Hash":
    "4a45186a93b8a2dd9301c66a3b3dad580b1bcfa74aadda583ca383f9fe0dea93"
    }
    },
    "iten": {
    "vocab": {
    "name": "vocab.iten.spm",
    "size": 814151,
    "estimatedCompressedSize": 405416,
    "expectedSha256Hash":
    "22d5ce6973be5360a921103acbe984a9bfca952a1f6c55c9cb5ef7de4fd58266"
    },
    "lex": {
    "name": "lex.50.50.iten.s2t.bin",
    "size": 5238420,
    "estimatedCompressedSize": 2860178,
    "expectedSha256Hash":
    "357d362373022b029ee9965975a133e6f36fdb0fed749202ff578365cf0111f8"
    },
    "model": {
    "name": "model.iten.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13423308,
    "expectedSha256Hash":
    "1fae546faeb9046f80b1b7e940b37b660974ce72902778181d6cd1c30b717f35"
    }
    },
    "pten": {
    "vocab": {
    "name": "vocab.pten.spm",
    "size": 812889,
    "estimatedCompressedSize": 406730,
    "expectedSha256Hash":
    "8389979e3c965688b07aeb712a7e44406e5dcdb2b84087229d26fcc71448c4ed"
    },
    "lex": {
    "name": "lex.50.50.pten.s2t.bin",
    "size": 5001420,
    "estimatedCompressedSize": 2733800,
    "expectedSha256Hash":
    "212ed0ae44a6f920cd6d17ca02f0a523ba6c4b0ef5078ae310c20bc4c51484c5"
    },
    "model": {
    "name": "model.pten.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13584764,
    "expectedSha256Hash":
    "6c3b7af01772022a19712410c63342ba581468c2f1aac34d7488409c4043e697"
    }
    },
    "ruen": {
    "vocab": {
    "name": "vocab.ruen.spm",
    "size": 936576,
    "estimatedCompressedSize": 435801,
    "expectedSha256Hash":
    "aaf9a325c0a988c507d0312cb6ba1a02bac7a370bcd879aedee626a40bfbda78"
    },
    "lex": {
    "name": "lex.50.50.ruen.s2t.bin",
    "size": 5090836,
    "estimatedCompressedSize": 2684919,
    "expectedSha256Hash":
    "e6667e22f5f86be4872e3768b7184727f5dd8c9f2ccfb0639baabcb1176f5d11"
    },
    "model": {
    "name": "model.ruen.intgemm.alphas.bin",
    "size": 17140836,
    "estimatedCompressedSize": 13108893,
    "expectedSha256Hash":
    "3b6a0305e3d232fadd54f5a765365b7b96ad6d8f2e818cba594b02fbd8fadb3d"
    }
    },
    "csen": {
    "vocab": {
    "name": "vocab.csen.spm",
    "size": 769763,
    "estimatedCompressedSize": 366392,
    "expectedSha256Hash":
    "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3"
    },
    "lex": {
    "name": "lex.50.50.csen.s2t.bin",
    "size": 4535788,
    "estimatedCompressedSize": 2418488,
    "expectedSha256Hash":
    "8228a3c3f7887759a62b7d7c674a7bef9b70161913f9b0939ab58f71186835c2"
    },
    "model": {
    "name": "model.csen.intgemm.alphas.bin",
    "size": 17140756,
    "estimatedCompressedSize": 13045032,
    "expectedSha256Hash":
    "5b16661e2864dc50b2f4091a16bdd4ec8d8283e04271e602159ba348df5d6e2d"
    }
    },
    "deen": {
    "vocab": {
    "name": "vocab.deen.spm",
    "size": 784269,
    "estimatedCompressedSize": 410738,
    "expectedSha256Hash":
    "417668f2ed297970febafb5b079a9d5ebc4ed0b3550ac8386d67a90473a09bd7"
    },
    "lex": {
    "name": "lex.50.50.deen.s2t.bin",
    "size": 5047568,
    "estimatedCompressedSize": 2657472,
    "expectedSha256Hash":
    "2f7c0f7bbce97ae5b52454074a892ba7b7610fb98e3c5d341e4ca79f0850c4de"
    },
    "model": {
    "name": "model.deen.intgemm.alphas.bin",
    "size": 17140837,
    "estimatedCompressedSize": 13091214,
    "expectedSha256Hash":
    "dda44d87ab0d8ad3b3871122fd3ee385f37878183a8b4ec139cd909531ec5009"
    }
    },
    "encs": {
    "vocab": {
    "name": "vocab.csen.spm",
    "size": 769763,
    "estimatedCompressedSize": 366392,
    "expectedSha256Hash":
    "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3"
    },
    "lex": {
    "name": "lex.50.50.encs.s2t.bin",
    "size": 3556124,
    "estimatedCompressedSize": 1913246,
    "expectedSha256Hash":
    "e19c77231bf977988e31ff8db15fe79966b5170564bd3e10613f239e7f461d97"
    },
    "model": {
    "name": "model.encs.intgemm.alphas.bin",
    "size": 17140756,
    "estimatedCompressedSize": 12630325,
    "expectedSha256Hash":
    "9a2fe0588bd972accfc801e2f31c945de0557804a91666ae5ab43b94fb74ac4b"
    }
    },
    "ende": {
    "vocab": {
    "name": "vocab.deen.spm",
    "size": 797501,
    "estimatedCompressedSize": 412505,
    "expectedSha256Hash":
    "bc8f8229933d8294c727f3eab12f6f064e7082b929f2d29494c8a1e619ba174c"
    },
    "lex": {
    "name": "lex.50.50.ende.s2t.bin",
    "size": 3062492,
    "estimatedCompressedSize": 1575385,
    "expectedSha256Hash":
    "764797d075f0642c0b079cce6547348d65fe4e92ac69fa6a8605cd8b53dacb3f"
    },
    "model": {
    "name": "model.ende.intgemm.alphas.bin",
    "size": 17140498,
    "estimatedCompressedSize": 13207068,
    "expectedSha256Hash":
    "f0946515c6645304f0706fa66a051c3b7b7c507f12d0c850f276c18165a10c14"
    }
    },
    "enes": {
    "vocab": {
    "name": "vocab.esen.spm",
    "size": 825463,
    "estimatedCompressedSize": 414566,
    "expectedSha256Hash":
    "909b1eea1face0d7f90a474fe29a8c0fef8d104b6e41e65616f864c964ba8845"
    },
    "lex": {
    "name": "lex.50.50.enes.s2t.bin",
    "size": 3347104,
    "estimatedCompressedSize": 1720700,
    "expectedSha256Hash":
    "3a113d713dec3cf1d12bba5b138ae616e28bba4bbc7fe7fd39ba145e26b86d7f"
    },
    "model": {
    "name": "model.enes.intgemm.alphas.bin",
    "size": 17140755,
    "estimatedCompressedSize": 12602853,
    "expectedSha256Hash":
    "fa7460037a3163e03fe1d23602f964bff2331da6ee813637e092ddf37156ef53"
    }
    },
    "enet": {
    "vocab": {
    "name": "vocab.eten.spm",
    "size": 828426,
    "estimatedCompressedSize": 416995,
    "expectedSha256Hash":
    "e3b66bc141f6123cd40746e2fb9b8ee4f89cbf324ab27d6bbf3782e52f15fa2d"
    },
    "lex": {
    "name": "lex.50.50.enet.s2t.bin",
    "size": 2700780,
    "estimatedCompressedSize": 1336443,
    "expectedSha256Hash":
    "3d1b40ff43ebef82cf98d416a88a1ea19eb325a85785eef102f59878a63a829d"
    },
    "model": {
    "name": "model.enet.intgemm.alphas.bin",
    "size": 17140754,
    "estimatedCompressedSize": 12543318,
    "expectedSha256Hash":
    "a28874a8b702a519a14dc71bcee726a5cb4b539eeaada2d06492f751469a1fd6"
    }
    },
    "esen": {
    "vocab": {
    "name": "vocab.esen.spm",
    "size": 825463,
    "estimatedCompressedSize": 414566,
    "expectedSha256Hash":
    "909b1eea1face0d7f90a474fe29a8c0fef8d104b6e41e65616f864c964ba8845"
    },
    "lex": {
    "name": "lex.50.50.esen.s2t.bin",
    "size": 3860888,
    "estimatedCompressedSize": 1978538,
    "expectedSha256Hash":
    "f11a2c23ef85ab1fee1c412b908d69bc20d66fd59faa8f7da5a5f0347eddf969"
    },
    "model": {
    "name": "model.esen.intgemm.alphas.bin",
    "size": 17140755,
    "estimatedCompressedSize": 13215960,
    "expectedSha256Hash":
    "4b6b7f451094aaa447d012658af158ffc708fc8842dde2f871a58404f5457fe0"
    }
    },
    "eten": {
    "vocab": {
    "name": "vocab.eten.spm",
    "size": 828426,
    "estimatedCompressedSize": 416995,
    "expectedSha256Hash":
    "e3b66bc141f6123cd40746e2fb9b8ee4f89cbf324ab27d6bbf3782e52f15fa2d"
    },
    "lex": {
    "name": "lex.50.50.eten.s2t.bin",
    "size": 3974944,
    "estimatedCompressedSize": 1920655,
    "expectedSha256Hash":
    "6992bedc590e60e610a28129c80746fe5f33144a4520e2c5508d87db14ca54f8"
    },
    "model": {
    "name": "model.eten.intgemm.alphas.bin",
    "size": 17140754,
    "estimatedCompressedSize": 12222624,
    "expectedSha256Hash":
    "aac98a2371e216ee2d4843cbe896c617f6687501e17225ac83482eba52fd0028"
    }
    }
    }
    }