Last active
April 4, 2017 01:17
-
-
Save jdanbrown/c08c60d388b16d6052b4545ab791a9af to your computer and use it in GitHub Desktop.
bq: unnest array vs. union select
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# TODO(db): Figure out if either of these is better than the other | |
# - Both give "query resources exceeded" on what_is_junk with all of the homes/addresses fields | |
def template_unnest_array(table, fields): | |
df = bq.Query(with_od_homes(''' | |
select * from unnest([ | |
%(array_elems)s | |
]) | |
order by i | |
''' % dict( | |
array_elems=',\n'.join([ | |
''' | |
struct( | |
%(i)s as i, | |
... | |
) | |
''' % dict( | |
i=i, | |
... | |
) | |
for i, ... in enumerate(...) | |
]), | |
))).to_dataframe() | |
df = df.drop('i', axis=1) | |
return df | |
def template_union_select(table, fields): | |
df = bq.Query(with_od_homes(''' | |
%(selects)s | |
order by i | |
''' % dict( | |
selects='\nunion all\n'.join([ | |
''' | |
select | |
%(i)s as i, | |
... | |
from %(table)s | |
''' % dict( | |
i=i, | |
... | |
) | |
for i, ... in enumerate(...) | |
]) | |
))).to_dataframe() | |
df = df.drop('i', axis=1) | |
return df |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment