Created
August 30, 2018 19:33
-
-
Save boppreh/a9737acb2abf015e6e828277b40efe71 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
golden_ratio = 0x9E3779B9 | |
max_int = 2**32 - 1 | |
def rotate_left_5(value): | |
return ((value << 5) | (value >> 27)) & max_int | |
def add_to_hash(hash_value, value): | |
return (golden_ratio * (rotate_left_5(hash_value) ^ value)) & max_int | |
def hash_simple(url): | |
hash_value = 0 | |
for char in url.encode('utf-8'): | |
hash_value = add_to_hash(hash_value, char) | |
return hash_value | |
def url_hash(url): | |
prefix, _ = url.split(':', 1) | |
return ((hash_simple(prefix) & 0x0000FFFF) << 32) + hash_simple(url) | |
if __name__ == '__main__': | |
assert url_hash("http://example.org/") == 125508604170377 | |
assert url_hash("https://example.org/") == 47358175329495 | |
assert url_hash("https://www.reddit.com/") == 47359719085711 | |
assert url_hash("https://old.reddit.com/") == 47358033120677 | |
assert url_hash("https://en.wikipedia.org/wiki/Libert%C3%A9") == 47359238090423 |
Thank you, this worked perfectly fine.
Instead of your example I used it in conjunction with this script.
# save as url_fix.py
from url_hash import url_hash
import sqlite3
import argparse
def url_fix(conn):
conn.create_function("urlhash",1,url_hash)
c = conn.cursor()
c.execute("UPDATE moz_places SET url_hash = urlhash(url) WHERE urlhash(url) <> url_hash")
conn.commit()
if __name__=="__main__":
parser = argparse.ArgumentParser()
parser.add_argument("conn",help="Insert the conn of the DB to update")
conn = parser.parse_args().conn
url_fix(sqlite3.connect(conn))
You can then use url_fix.py places.sqlite
.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This is a Python port of https://github.com/bencaradocdavies/sqlite-mozilla-url-hash .
url_hash
calculates the Firefox 50+ function used inplaces.sqlite
, and necessary when inserting or editing history elements in that database.Example usage: