Skip to content

Instantly share code, notes, and snippets.

@danielbraun

danielbraun/Makefile

Last active April 17, 2020 16:08
Download ZIP

Revisions

  1. danielbraun revised this gist Apr 17, 2020. 1 changed file with 42 additions and 15 deletions.
    57 changes: 42 additions & 15 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,7 @@
    SPACY_MODEL ?= xx_ent_wiki_sm
    PYTHONPATH ?= ~/django_home
    DJANGO_SETTINGS_MODULE ?= django_home.settings

    %.min.js: %.js
    uglifyjs < $< > $@

    @@ -10,6 +14,19 @@
    %.tab: %.mysql.sql
    mysql --defaults-file=my.cnf < $< > $@

    %.xml: %.mysql.sql
    mysql --defaults-file=my.cnf --xml < $< > $@

    %.html: %.psql.sql
    psql --html < $< > $@

    %.xml: %.psql.sql
    # TODO fix double quotes
    echo "select concat('<?xml version=\"1.0\"?>',query_to_xml(:q, false, false, ''));" \
    | psql -t -A -v q="'$(shell cat $<)'" \
    > $@


    %.xml: %.html
    xsltproc --html $*.xsl $< -o $@

    @@ -25,6 +42,12 @@
    %.out: %.curl
    curl -K $< > $@

    %.models.py:
    PYTHONPATH=$(PYTHONPATH) DJANGO_SETTINGS_MODULE=$(DJANGO_SETTINGS_MODULE) python3 -m django inspectdb $(shell basename $*) > $@

    U.S.DividendChampions.xlsx:
    curl -L https://bitly.com/USDividendChampions > $@

    %.dividends.csv:
    curl "https://query1.finance.yahoo.com/v7/finance/download/$(shell basename $*)?period1=0&period2=$(shell date +%s)&interval=1d&events=div" > $@

    @@ -149,27 +172,31 @@ airbnb/%:
    > $@

    %.spacy.json: %.txt
    python3 -c'import spacy, sys, json; print(json.dumps(spacy.load("outtt")("".join(sys.stdin.readlines())).to_json()))' < $< | jq > $@
    python3 -c'import spacy, sys, json; print(json.dumps(spacy.load("$(SPACY_MODEL)")("".join(sys.stdin.readlines())).to_json()))' < $< | jq > $@

    %.ent.spacy.html: %.txt
    python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("outtt/model-best")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@
    python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("$(SPACY_MODEL)")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@

    %.png: %.pdf
    cd $(shell dirname $@) && pdftocairo -png -singlefile $(shell basename $<)

    tasksheriff/oauth.json: tasksheriff/credentials.json
    curl "https://api.tasksheriff.com/oauth/token" \
    -H "Accept: application/json"\
    -H "Content-Type: application/json"\
    --data @$< \
    > $@
    dividend-aristocrats.csv:
    curl -L http://www.simplysafedividends.com/intelligent-income/idea_lists/3-dividend-aristocrats.csv > $@

    tasksheriff/access_token: tasksheriff/oauth.json
    jq -r ".access_token" < $< > $@
    %.test.iob: %.iob
    cat $< | head -n $(shell echo $(shell cat $< | wc -l) \* 0.6 / 1 | bc) > $@

    tasksheriff/files/%.pdf: tasksheriff/access_token
    curl https://api.tasksheriff.com/api/download/$* \
    -H "Authorization: Bearer $(shell cat $<)" \
    > $@
    %.dev.iob: %.iob
    cat $< | head -n $(shell echo $(shell cat $< | wc -l) \* 0.4 / 1 | bc) > $@

    %.model: %.test.json %.dev.json
    rm -rf $@
    python3 -m spacy train \
    -p ner \
    -b xx_ent_wiki_sm\
    xx $@ $*.test.json $*.dev.json

    clean:
    git clean -fdX

    .PHONY: %/dividends *.php
    .PHONY: %/dividends *.php clean
  2. danielbraun revised this gist Apr 5, 2020. 1 changed file with 17 additions and 5 deletions.
    22 changes: 17 additions & 5 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -50,6 +50,9 @@
    %.csv: %.json
    json2csv < $< > $@

    %.json: %.iob
    python3 -m spacy convert $< > $@

    %.table.txt: %.csv
    csvlook $@ > $@

    @@ -112,20 +115,29 @@ domains/%:
    nslookup $* >> $@

    %.svg: %.geojson
    gso2svg < $< > $@
    geo2svg < $< > $@

    %.kml: %.geojson
    ./node_modules/.bin/tokml $< > $@

    server:
    shell2http -cgi / 'export FILE=.`echo $$REQUEST_URI | sed "s/?.*//"` && make -s $$FILE && cat $$FILE'

    Corona_Exposure_View.geojson:
    curl "https://services5.arcgis.com/dlrDjz89gx9qyfev/arcgis/rest/services/Corona_Exposure_View/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=102100&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=false&cacheHint=true&orderByFields=&groupByFieldsForStatistics=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=%7B%22mode%22%3A%22view%22%2C%22originPosition%22%3A%22upperLeft%22%2C%22tolerance%22%3A1.0583333333333333%2C%22extent%22%3A%7B%22xmin%22%3A34.526090200000056%2C%22ymin%22%3A29.54879070000004%2C%22xmax%22%3A35.779580500000066%2C%22ymax%22%3A33.20352598000005%2C%22spatialReference%22%3A%7B%22wkid%22%3A4326%2C%22latestWkid%22%3A4326%7D%7D%7D&sqlFormat=none&f=geojson" -o $@
    corona.geojson:
    curl "https://services5.arcgis.com/dlrDjz89gx9qyfev/arcgis/rest/services/Corona_Exposure_View/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=false&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pgeojson&token=" -o $@

    %.leaflet.json: %.geojson
    jq -s ".[0].geoJSON=.[1] | first" leaflet.json $< > $@

    %.json: %.xml
    xml2json < $< > $@

    %.json.js: %.json
    browserify --require ./$< -s $(shell basename $*) > $@

    %.json.xml: %.json
    echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"leaflet.xsl\" type=\"text/xsl\"?><root><![CDATA[$$(cat $<)]]></root>" > $@

    %.leaflet.html: $.leaflet.json

    %.sql: %.csv
    @@ -137,10 +149,10 @@ airbnb/%:
    > $@

    %.spacy.json: %.txt
    python3 -c'import spacy, sys, json; print(json.dumps(spacy.load("xx_ent_wiki_sm")("".join(sys.stdin.readlines())).to_json()))' < $< | jq > $@
    python3 -c'import spacy, sys, json; print(json.dumps(spacy.load("outtt")("".join(sys.stdin.readlines())).to_json()))' < $< | jq > $@

    %.ent.spacy.html: %.txt
    python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("xx_ent_wiki_sm")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@
    python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("outtt/model-best")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@

    %.png: %.pdf
    cd $(shell dirname $@) && pdftocairo -png -singlefile $(shell basename $<)
  3. danielbraun revised this gist Apr 2, 2020. 1 changed file with 19 additions and 1 deletion.
    20 changes: 19 additions & 1 deletion Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -8,7 +8,7 @@
    browserify $< > $@

    %.tab: %.mysql.sql
    mysql --defaults-extra-file=my.cnf < $< > $@
    mysql --defaults-file=my.cnf < $< > $@

    %.xml: %.html
    xsltproc --html $*.xsl $< -o $@
    @@ -142,4 +142,22 @@ airbnb/%:
    %.ent.spacy.html: %.txt
    python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("xx_ent_wiki_sm")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@

    %.png: %.pdf
    cd $(shell dirname $@) && pdftocairo -png -singlefile $(shell basename $<)

    tasksheriff/oauth.json: tasksheriff/credentials.json
    curl "https://api.tasksheriff.com/oauth/token" \
    -H "Accept: application/json"\
    -H "Content-Type: application/json"\
    --data @$< \
    > $@

    tasksheriff/access_token: tasksheriff/oauth.json
    jq -r ".access_token" < $< > $@

    tasksheriff/files/%.pdf: tasksheriff/access_token
    curl https://api.tasksheriff.com/api/download/$* \
    -H "Authorization: Bearer $(shell cat $<)" \
    > $@

    .PHONY: %/dividends *.php
  4. danielbraun revised this gist Apr 2, 2020. 1 changed file with 39 additions and 5 deletions.
    44 changes: 39 additions & 5 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,3 @@
    all:

    %.min.js: %.js
    uglifyjs < $< > $@

    @@ -69,13 +67,12 @@ all:
    %.svg: %.csv
    gnuplot -e "set key autotitle columnhead; set datafile separator ','; set terminal svg enhanced mouse jsdir 'http://gnuplot.sourceforge.net/demo_svg_5.2'; plot '$<';" > $@

    http://gnuplot.sourceforge.net/demo_svg_5.2/gnuplot_svg.js

    %.instagram.jpg:
    curl -L "https://www.instagram.com/p/$(shell basename $*)/media?size=l" > $@

    # Sign in doesn't work
    %.album.html:
    music/%:
    @mkdir -p `dirname $@`
    printf "<iframe \
    src='https://embed.music.apple.com/us/album/$(shell basename $*)' \
    height='100%%' \
    @@ -85,6 +82,15 @@ http://gnuplot.sourceforge.net/demo_svg_5.2/gnuplot_svg.js
    width='100%%' \
    ></iframe>" > $@

    youtube/%:
    @mkdir -p `dirname $@`
    printf "<iframe \
    src='https://www.youtube.com/embed/$(shell basename $*)' \
    height='100%%' \
    frameborder='0' \
    width='100%%' \
    ></iframe>" > $@

    yad2/rentals/%.json:
    @mkdir -p `dirname $@`
    @res="$(shell curl 'https://www.yad2.co.il/api/pre-load/getFeedIndex/realestate/rent?page=$*' \
    @@ -105,7 +111,35 @@ domains/%:
    whois $* > $@
    nslookup $* >> $@

    %.svg: %.geojson
    gso2svg < $< > $@

    server:
    shell2http -cgi / 'export FILE=.`echo $$REQUEST_URI | sed "s/?.*//"` && make -s $$FILE && cat $$FILE'

    Corona_Exposure_View.geojson:
    curl "https://services5.arcgis.com/dlrDjz89gx9qyfev/arcgis/rest/services/Corona_Exposure_View/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=102100&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=false&cacheHint=true&orderByFields=&groupByFieldsForStatistics=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=%7B%22mode%22%3A%22view%22%2C%22originPosition%22%3A%22upperLeft%22%2C%22tolerance%22%3A1.0583333333333333%2C%22extent%22%3A%7B%22xmin%22%3A34.526090200000056%2C%22ymin%22%3A29.54879070000004%2C%22xmax%22%3A35.779580500000066%2C%22ymax%22%3A33.20352598000005%2C%22spatialReference%22%3A%7B%22wkid%22%3A4326%2C%22latestWkid%22%3A4326%7D%7D%7D&sqlFormat=none&f=geojson" -o $@

    %.leaflet.json: %.geojson
    jq -s ".[0].geoJSON=.[1] | first" leaflet.json $< > $@

    %.json: %.xml
    xml2json < $< > $@

    %.leaflet.html: $.leaflet.json

    %.sql: %.csv
    csvsql $< > $@

    airbnb/%:
    curl "https://www.airbnb.com/api/v2/pdp_listing_booking_details?_format=for_web_with_date&check_in=2020-03-29&check_out=2020-04-21&currency=ILS&guests=1&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&listing_id=$*&locale=en&number_of_adults=1&number_of_children=0&number_of_infants=0" \
    | jq ".pdp_listing_booking_details[0].price.total.amount" \
    > $@

    %.spacy.json: %.txt
    python3 -c'import spacy, sys, json; print(json.dumps(spacy.load("xx_ent_wiki_sm")("".join(sys.stdin.readlines())).to_json()))' < $< | jq > $@

    %.ent.spacy.html: %.txt
    python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("xx_ent_wiki_sm")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@

    .PHONY: %/dividends *.php
  5. danielbraun revised this gist Mar 25, 2020. 1 changed file with 62 additions and 3 deletions.
    65 changes: 62 additions & 3 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,5 @@
    all:

    %.min.js: %.js
    uglifyjs < $< > $@

    @@ -10,6 +12,21 @@
    %.tab: %.mysql.sql
    mysql --defaults-extra-file=my.cnf < $< > $@

    %.xml: %.html
    xsltproc --html $*.xsl $< -o $@

    %.txt: %.pdf
    pdftotext - - < $< > $@

    %.info.txt: %.pdf
    pdfinfo $< > $@

    %.pdf: %.svg
    cairosvg $< -o $@

    %.out: %.curl
    curl -K $< > $@

    %.dividends.csv:
    curl "https://query1.finance.yahoo.com/v7/finance/download/$(shell basename $*)?period1=0&period2=$(shell date +%s)&interval=1d&events=div" > $@

    @@ -35,6 +52,9 @@
    %.csv: %.json
    json2csv < $< > $@

    %.table.txt: %.csv
    csvlook $@ > $@

    %.json: %.min.json
    jq < $< > $@

    @@ -47,6 +67,45 @@
    php < $< > $@

    %.svg: %.csv
    gnuplot -e "set key autotitle columnhead; set datafile separator ','; set terminal svg; plot '$<';" > $@

    .PHONY: %/dividends
    gnuplot -e "set key autotitle columnhead; set datafile separator ','; set terminal svg enhanced mouse jsdir 'http://gnuplot.sourceforge.net/demo_svg_5.2'; plot '$<';" > $@

    http://gnuplot.sourceforge.net/demo_svg_5.2/gnuplot_svg.js

    %.instagram.jpg:
    curl -L "https://www.instagram.com/p/$(shell basename $*)/media?size=l" > $@

    # Sign in doesn't work
    %.album.html:
    printf "<iframe \
    src='https://embed.music.apple.com/us/album/$(shell basename $*)' \
    height='100%%' \
    frameborder='0' \
    allow='autoplay *; encrypted-media *;' \
    sandbox='allow-forms allow-popups allow-same-origin allow-scripts allow-storage-access-by-user-activation allow-top-navigation-by-user-activation' \
    width='100%%' \
    ></iframe>" > $@

    yad2/rentals/%.json:
    @mkdir -p `dirname $@`
    @res="$(shell curl 'https://www.yad2.co.il/api/pre-load/getFeedIndex/realestate/rent?page=$*' \
    -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/$*.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15' \
    -f \
    -L \
    -s \
    -w %{url_effective} \
    -o $@)" \
    ; file --mime $@ | grep "text/html" \
    && echo "Error: Verification required" \
    && rm $@ && open $$res\
    && exit 1 \
    || exit 0

    domains/%:
    mkdir -p $(shell dirname $@)
    whois $* > $@
    nslookup $* >> $@

    server:
    shell2http -cgi / 'export FILE=.`echo $$REQUEST_URI | sed "s/?.*//"` && make -s $$FILE && cat $$FILE'

    .PHONY: %/dividends *.php
  6. danielbraun created this gist Mar 23, 2020.
    52 changes: 52 additions & 0 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,52 @@
    %.min.js: %.js
    uglifyjs < $< > $@

    %.xml: %.edn
    edn2xml < $< > $@

    %.bundle.js: %.js
    browserify $< > $@

    %.tab: %.mysql.sql
    mysql --defaults-extra-file=my.cnf < $< > $@

    %.dividends.csv:
    curl "https://query1.finance.yahoo.com/v7/finance/download/$(shell basename $*)?period1=0&period2=$(shell date +%s)&interval=1d&events=div" > $@

    %.sorted.csv: %.csv
    csvsort < $< > $@

    %/stocks.nyse.min.json:
    curl 'https://www.nyse.com/api/quotes/filter' \
    -XPOST \
    -H 'Content-Type: application/json' \
    --data-binary '{"instrumentType":"EQUITY","pageNumber":1,"maxResultsPerPage":100000}' > $@

    %/dividends: %/stock_symbols.nyse.csv.tail
    mkdir -p $@
    cat $< | xargs printf "$@/%s.dividends.sorted.csv\n" | xargs $(MAKE)

    %/stock_symbols.nyse.csv: %/stocks.nyse.csv
    cat $< | csvcut -c symbolTicker > $@

    %.tail: %
    tail -n +2 < $< > $@

    %.csv: %.json
    json2csv < $< > $@

    %.json: %.min.json
    jq < $< > $@

    %.min.json: %.csv
    csvjson < $< > $@

    %.out.edn: %.edn

    %.xml: %.php
    php < $< > $@

    %.svg: %.csv
    gnuplot -e "set key autotitle columnhead; set datafile separator ','; set terminal svg; plot '$<';" > $@

    .PHONY: %/dividends