Last active
August 9, 2021 12:23
-
-
Save flare9x/a365cce7ab935c5705bad3cbc47edc06 to your computer and use it in GitHub Desktop.
NOAA GHCN data is fixed width data - this parses the .dly files and exports as .csv
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Data: https://www.ncei.noaa.gov/access/metadata/landing-page/bin/iso?id=gov.noaa.ncdc%3AC00861 | |
| ###################################################################################################### | |
| # load .dly data file | |
| # load as 1x string per row | |
| # split the string per the readme .txt element / column position (static) | |
| # export as .csv | |
| ###################################################################################################### | |
| # read all files in directory | |
| all_dly_files = cd(readdir, "C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all/") | |
| all_dly_files_for_export = chop.(all_dly_files, tail = 4) | |
| # initialize output | |
| all_out_tmax = DataFrame(Date = Date[], TMAX = Float64[]) | |
| all_out_tmin = DataFrame(Date = Date[], TMIN = Float64[]) | |
| all_out_tavg = DataFrame(Date = Date[], TAVG = Float64[]) | |
| # start loop | |
| d=1 | |
| for d = 1:length(all_dly_files) | |
| print("Starting dly file ", all_dly_files[d]," - Iteration ",d,"\n") | |
| data = CSV.read(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all/", all_dly_files[d]]), DataFrame, header=false,delim="", ignorerepeated=false) | |
| #data = CSV.read("C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all/ACW00011604.dly", DataFrame, header=false,delim="", ignorerepeated=false) | |
| ID = fill("",size(data,1)) | |
| YEAR = fill("",size(data,1)) | |
| MONTH = fill("",size(data,1)) | |
| ELEMENT = fill("",size(data,1)) | |
| VALUE1 = fill(0.0,size(data,1)) | |
| MFLAG1 = fill("",size(data,1)) | |
| QFLAG1 = fill("",size(data,1)) | |
| SFLAG1 = fill("",size(data,1)) | |
| VALUE2 = fill(0.0,size(data,1)) | |
| MFLAG2 = fill("",size(data,1)) | |
| QFLAG2 = fill("",size(data,1)) | |
| SFLAG2 = fill("",size(data,1)) | |
| VALUE3 = fill(0.0,size(data,1)) | |
| MFLAG3 = fill("",size(data,1)) | |
| QFLAG3 = fill("",size(data,1)) | |
| SFLAG3 = fill("",size(data,1)) | |
| VALUE4 = fill(0.0,size(data,1)) | |
| MFLAG4 = fill("",size(data,1)) | |
| QFLAG4 = fill("",size(data,1)) | |
| SFLAG4 = fill("",size(data,1)) | |
| VALUE5 = fill(0.0,size(data,1)) | |
| MFLAG5 = fill("",size(data,1)) | |
| QFLAG5 = fill("",size(data,1)) | |
| SFLAG5 = fill("",size(data,1)) | |
| VALUE6 = fill(0.0,size(data,1)) | |
| MFLAG6 = fill("",size(data,1)) | |
| QFLAG6 = fill("",size(data,1)) | |
| SFLAG6 = fill("",size(data,1)) | |
| VALUE7 = fill(0.0,size(data,1)) | |
| MFLAG7 = fill("",size(data,1)) | |
| QFLAG7 = fill("",size(data,1)) | |
| SFLAG7 = fill("",size(data,1)) | |
| VALUE8 = fill(0.0,size(data,1)) | |
| MFLAG8 = fill("",size(data,1)) | |
| QFLAG8 = fill("",size(data,1)) | |
| SFLAG8 = fill("",size(data,1)) | |
| VALUE9 = fill(0.0,size(data,1)) | |
| MFLAG9 = fill("",size(data,1)) | |
| QFLAG9 = fill("",size(data,1)) | |
| SFLAG9 = fill("",size(data,1)) | |
| VALUE10 = fill(0.0,size(data,1)) | |
| MFLAG10 = fill("",size(data,1)) | |
| QFLAG10 = fill("",size(data,1)) | |
| SFLAG10 = fill("",size(data,1)) | |
| VALUE11 = fill(0.0,size(data,1)) | |
| MFLAG11 = fill("",size(data,1)) | |
| QFLAG11 = fill("",size(data,1)) | |
| SFLAG11 = fill("",size(data,1)) | |
| VALUE12 = fill(0.0,size(data,1)) | |
| MFLAG12 = fill("",size(data,1)) | |
| QFLAG12 = fill("",size(data,1)) | |
| SFLAG12 = fill("",size(data,1)) | |
| VALUE13 = fill(0.0,size(data,1)) | |
| MFLAG13 = fill("",size(data,1)) | |
| QFLAG13 = fill("",size(data,1)) | |
| SFLAG13 = fill("",size(data,1)) | |
| VALUE14 = fill(0.0,size(data,1)) | |
| MFLAG14 = fill("",size(data,1)) | |
| QFLAG14 = fill("",size(data,1)) | |
| SFLAG14 = fill("",size(data,1)) | |
| VALUE15 = fill(0.0,size(data,1)) | |
| MFLAG15 = fill("",size(data,1)) | |
| QFLAG15 = fill("",size(data,1)) | |
| SFLAG15 = fill("",size(data,1)) | |
| VALUE16 = fill(0.0,size(data,1)) | |
| MFLAG16 = fill("",size(data,1)) | |
| QFLAG16 = fill("",size(data,1)) | |
| SFLAG16 = fill("",size(data,1)) | |
| VALUE17 = fill(0.0,size(data,1)) | |
| MFLAG17 = fill("",size(data,1)) | |
| QFLAG17 = fill("",size(data,1)) | |
| SFLAG17 = fill("",size(data,1)) | |
| VALUE18 = fill(0.0,size(data,1)) | |
| MFLAG18 = fill("",size(data,1)) | |
| QFLAG18 = fill("",size(data,1)) | |
| SFLAG18 = fill("",size(data,1)) | |
| VALUE19 = fill(0.0,size(data,1)) | |
| MFLAG19 = fill("",size(data,1)) | |
| QFLAG19 = fill("",size(data,1)) | |
| SFLAG19 = fill("",size(data,1)) | |
| VALUE20 = fill(0.0,size(data,1)) | |
| MFLAG20 = fill("",size(data,1)) | |
| QFLAG20 = fill("",size(data,1)) | |
| SFLAG20 = fill("",size(data,1)) | |
| VALUE21 = fill(0.0,size(data,1)) | |
| MFLAG21 = fill("",size(data,1)) | |
| QFLAG21 = fill("",size(data,1)) | |
| SFLAG21 = fill("",size(data,1)) | |
| VALUE22 = fill(0.0,size(data,1)) | |
| MFLAG22 = fill("",size(data,1)) | |
| QFLAG22 = fill("",size(data,1)) | |
| SFLAG22 = fill("",size(data,1)) | |
| VALUE23 = fill(0.0,size(data,1)) | |
| MFLAG23 = fill("",size(data,1)) | |
| QFLAG23 = fill("",size(data,1)) | |
| SFLAG23 = fill("",size(data,1)) | |
| VALUE24 = fill(0.0,size(data,1)) | |
| MFLAG24 = fill("",size(data,1)) | |
| QFLAG24 = fill("",size(data,1)) | |
| SFLAG24 = fill("",size(data,1)) | |
| VALUE25 = fill(0.0,size(data,1)) | |
| MFLAG25 = fill("",size(data,1)) | |
| QFLAG25 = fill("",size(data,1)) | |
| SFLAG25 = fill("",size(data,1)) | |
| VALUE26 = fill(0.0,size(data,1)) | |
| MFLAG26 = fill("",size(data,1)) | |
| QFLAG26 = fill("",size(data,1)) | |
| SFLAG26 = fill("",size(data,1)) | |
| VALUE27 = fill(0.0,size(data,1)) | |
| MFLAG27 = fill("",size(data,1)) | |
| QFLAG27 = fill("",size(data,1)) | |
| SFLAG27 = fill("",size(data,1)) | |
| VALUE28 = fill(0.0,size(data,1)) | |
| MFLAG28 = fill("",size(data,1)) | |
| QFLAG28 = fill("",size(data,1)) | |
| SFLAG28 = fill("",size(data,1)) | |
| VALUE29 = fill(0.0,size(data,1)) | |
| MFLAG29 = fill("",size(data,1)) | |
| QFLAG29 = fill("",size(data,1)) | |
| SFLAG29 = fill("",size(data,1)) | |
| VALUE30 = fill(0.0,size(data,1)) | |
| MFLAG30 = fill("",size(data,1)) | |
| QFLAG30 = fill("",size(data,1)) | |
| SFLAG30 = fill("",size(data,1)) | |
| VALUE31 = fill(0.0,size(data,1)) | |
| MFLAG31 = fill("",size(data,1)) | |
| QFLAG31 = fill("",size(data,1)) | |
| SFLAG31 = fill("",size(data,1)) | |
| i =1 | |
| for i = 1:size(data,1) | |
| temp = data[i,1] | |
| ID[i] = temp[1:11] | |
| YEAR[i] = temp[12:15] | |
| MONTH[i] = temp[16:17] | |
| ELEMENT[i] = temp[18:21] | |
| VALUE1[i] = parse.(Float64, temp[22:26]) / 10 # day 1 of the month | |
| MFLAG1[i] = string(temp[27]) | |
| QFLAG1[i] = string(temp[28]) | |
| SFLAG1[i] = string(temp[29]) | |
| VALUE2[i] = parse.(Float64, temp[30:34]) / 10 # day 2 of the month | |
| MFLAG2[i] = string(temp[35]) | |
| QFLAG2[i] = string(temp[36]) | |
| SFLAG2[i] = string(temp[37]) | |
| VALUE3[i] = parse.(Float64, temp[38:42]) / 10 # day 3 of the month | |
| MFLAG3[i] = string(temp[43]) | |
| QFLAG3[i] = string(temp[44]) | |
| SFLAG3[i] = string(temp[45]) | |
| VALUE4[i] = parse.(Float64, temp[46:50]) / 10 # day 4 of the month | |
| MFLAG4[i] = string(temp[51]) | |
| QFLAG4[i] = string(temp[52]) | |
| SFLAG4[i] = string(temp[53]) | |
| VALUE5[i] = parse.(Float64, temp[54:58]) / 10 # day 5 of the month | |
| MFLAG5[i] = string(temp[59]) | |
| QFLAG5[i] = string(temp[60]) | |
| SFLAG5[i] = string(temp[61]) | |
| VALUE6[i] = parse.(Float64, temp[62:66]) / 10 # day 6 of the month | |
| MFLAG6[i] = string(temp[67]) | |
| QFLAG6[i] = string(temp[68]) | |
| SFLAG6[i] = string(temp[69]) | |
| VALUE7[i] = parse.(Float64, temp[70:74]) / 10 # day 7 of the month | |
| MFLAG7[i] = string(temp[75]) | |
| QFLAG7[i] = string(temp[76]) | |
| SFLAG7[i] = string(temp[77]) | |
| VALUE8[i] = parse.(Float64, temp[78:82]) / 10 # day 8 of the month | |
| MFLAG8[i] = string(temp[83]) | |
| QFLAG8[i] = string(temp[84]) | |
| SFLAG8[i] = string(temp[85]) | |
| VALUE9[i] = parse.(Float64, temp[86:90]) / 10 # day 9 of the month | |
| MFLAG9[i] = string(temp[91]) | |
| QFLAG9[i] = string(temp[92]) | |
| SFLAG9[i] = string(temp[93]) | |
| VALUE10[i] = parse.(Float64, temp[94:98]) / 10 # day 10 of the month | |
| MFLAG10[i] = string(temp[99]) | |
| QFLAG10[i] = string(temp[100]) | |
| SFLAG10[i] = string(temp[101]) | |
| VALUE11[i] = parse.(Float64, temp[102:106]) / 10 # day 11 of the month | |
| MFLAG11[i] = string(temp[107]) | |
| QFLAG11[i] = string(temp[108]) | |
| SFLAG11[i] = string(temp[109]) | |
| VALUE12[i] = parse.(Float64, temp[110:114]) / 10 # day 12 of the month | |
| MFLAG12[i] = string(temp[115]) | |
| QFLAG12[i] = string(temp[116]) | |
| SFLAG12[i] = string(temp[117]) | |
| VALUE13[i] = parse.(Float64, temp[118:122]) / 10 # day 13 of the month | |
| MFLAG13[i] = string(temp[123]) | |
| QFLAG13[i] = string(temp[124]) | |
| SFLAG13[i] = string(temp[125]) | |
| VALUE14[i] = parse.(Float64, temp[126:130]) / 10 # day 14 of the month | |
| MFLAG14[i] = string(temp[131]) | |
| QFLAG14[i] = string(temp[132]) | |
| SFLAG14[i] = string(temp[133]) | |
| VALUE15[i] = parse.(Float64, temp[134:138]) / 10 # day 15 of the month | |
| MFLAG15[i] = string(temp[139]) | |
| QFLAG15[i] = string(temp[140]) | |
| SFLAG15[i] = string(temp[141]) | |
| VALUE16[i] = parse.(Float64, temp[142:146]) / 10 # day 16 of the month | |
| MFLAG16[i] = string(temp[147]) | |
| QFLAG16[i] = string(temp[148]) | |
| SFLAG16[i] = string(temp[149]) | |
| VALUE17[i] = parse.(Float64, temp[150:154]) / 10 # day 17 of the month | |
| MFLAG17[i] = string(temp[155]) | |
| QFLAG17[i] = string(temp[156]) | |
| SFLAG17[i] = string(temp[157]) | |
| VALUE18[i] = parse.(Float64, temp[158:162]) / 10 # day 18 of the month | |
| MFLAG18[i] = string(temp[163]) | |
| QFLAG18[i] = string(temp[164]) | |
| SFLAG18[i] = string(temp[165]) | |
| VALUE19[i] = parse.(Float64, temp[166:170]) / 10 # day 19 of the month | |
| MFLAG19[i] = string(temp[171]) | |
| QFLAG19[i] = string(temp[172]) | |
| SFLAG19[i] = string(temp[173]) | |
| VALUE20[i] = parse.(Float64, temp[174:178]) / 10 # day 20 of the month | |
| MFLAG20[i] = string(temp[179]) | |
| QFLAG20[i] = string(temp[180]) | |
| SFLAG20[i] = string(temp[181]) | |
| VALUE21[i] = parse.(Float64, temp[182:186]) / 10 # day 21 of the month | |
| MFLAG21[i] = string(temp[187]) | |
| QFLAG21[i] = string(temp[188]) | |
| SFLAG21[i] = string(temp[189]) | |
| VALUE22[i] = parse.(Float64, temp[190:194]) / 10 # day 22 of the month | |
| MFLAG22[i] = string(temp[195]) | |
| QFLAG22[i] = string(temp[196]) | |
| SFLAG22[i] = string(temp[197]) | |
| VALUE23[i] = parse.(Float64, temp[198:202]) / 10 # day 23 of the month | |
| MFLAG23[i] = string(temp[203]) | |
| QFLAG23[i] = string(temp[204]) | |
| SFLAG23[i] = string(temp[205]) | |
| VALUE24[i] = parse.(Float64, temp[206:210]) / 10 # day 24 of the month | |
| MFLAG24[i] = string(temp[211]) | |
| QFLAG24[i] = string(temp[212]) | |
| SFLAG24[i] = string(temp[213]) | |
| VALUE25[i] = parse.(Float64, temp[214:218]) / 10 / 10 # day 25 of the month | |
| MFLAG25[i] = string(temp[219]) | |
| QFLAG25[i] = string(temp[220]) | |
| SFLAG25[i] = string(temp[221]) | |
| VALUE26[i] = parse.(Float64, temp[222:226]) / 10 / 10 # day 26 of the month | |
| MFLAG26[i] = string(temp[227]) | |
| QFLAG26[i] = string(temp[228]) | |
| SFLAG26[i] = string(temp[229]) | |
| VALUE27[i] = parse.(Float64, temp[230:234]) / 10 # day 27 of the month | |
| MFLAG27[i] = string(temp[235]) | |
| QFLAG27[i] = string(temp[236]) | |
| SFLAG27[i] = string(temp[237]) | |
| VALUE28[i] = parse.(Float64, temp[238:242]) / 10 # day 28 of the month | |
| MFLAG28[i] = string(temp[243]) | |
| QFLAG28[i] = string(temp[244]) | |
| SFLAG28[i] = string(temp[245]) | |
| VALUE29[i] = parse.(Float64, temp[246:250]) / 10 # day 29 of the month | |
| MFLAG29[i] = string(temp[251]) | |
| QFLAG29[i] = string(temp[252]) | |
| SFLAG29[i] = string(temp[253]) | |
| VALUE30[i] = parse.(Float64, temp[254:258]) / 10 # day 30 of the month | |
| MFLAG30[i] = string(temp[259]) | |
| QFLAG30[i] = string(temp[260]) | |
| SFLAG30[i] = string(temp[261]) | |
| VALUE31[i] = parse.(Float64, temp[262:266]) / 10 # day 31 of the month | |
| MFLAG31[i] = string(temp[267]) | |
| QFLAG31[i] = string(temp[268]) | |
| SFLAG31[i] = string(temp[269]) | |
| end | |
| # column names | |
| colnames = ["ID","YEAR","MONTH","ELEMENT","VALUE1","MFLAG1","QFLAG1","SFLAG1","VALUE2","MFLAG2","QFLAG2","SFLAG2","VALUE3","MFLAG3","QFLAG3","SFLAG3","VALUE4","MFLAG4","QFLAG4","SFLAG4","VALUE5", | |
| "MFLAG5","QFLAG5","SFLAG5","VALUE6","MFLAG6","QFLAG6","SFLAG6","VALUE7","MFLAG7","QFLAG7","SFLAG7","VALUE8","MFLAG8","QFLAG8","SFLAG8","VALUE9","MFLAG9","QFLAG9","SFLAG9","VALUE10","MFLAG10", | |
| "QFLAG10","SFLAG10","VALUE11","MFLAG11","QFLAG11","SFLAG11","VALUE12","MFLAG12","QFLAG12","SFLAG12","VALUE13","MFLAG13","QFLAG13","SFLAG13","VALUE14","MFLAG14","QFLAG14","SFLAG14", | |
| "VALUE15","MFLAG15","QFLAG15","SFLAG15","VALUE16","MFLAG16","QFLAG16","SFLAG16","VALUE17","MFLAG17","QFLAG17","SFLAG17","VALUE18","MFLAG18","QFLAG18","SFLAG18","VALUE19","MFLAG19", | |
| "QFLAG19","SFLAG19","VALUE20","MFLAG20","QFLAG20","SFLAG20","VALUE21","MFLAG21","QFLAG21","SFLAG21","VALUE22","MFLAG22","QFLAG22","SFLAG22","VALUE23","MFLAG23","QFLAG23","SFLAG23", | |
| "VALUE24","MFLAG24","QFLAG24","SFLAG24","VALUE25","MFLAG25","QFLAG25","SFLAG25","VALUE26","MFLAG26","QFLAG26","SFLAG26","VALUE27","MFLAG27","QFLAG27","SFLAG27","VALUE28","MFLAG28", | |
| "QFLAG28","SFLAG28","VALUE29","MFLAG29","QFLAG29","SFLAG29","VALUE30","MFLAG30","QFLAG30","SFLAG30","VALUE31","MFLAG31","QFLAG31","SFLAG31"] | |
| # output data frame | |
| out_df = DataFrame(hcat(ID,YEAR,MONTH,ELEMENT,VALUE1,MFLAG1,QFLAG1,SFLAG1,VALUE2,MFLAG2,QFLAG2,SFLAG2,VALUE3,MFLAG3,QFLAG3,SFLAG3,VALUE4,MFLAG4,QFLAG4,SFLAG4,VALUE5, | |
| MFLAG5,QFLAG5,SFLAG5,VALUE6,MFLAG6,QFLAG6,SFLAG6,VALUE7,MFLAG7,QFLAG7,SFLAG7,VALUE8,MFLAG8,QFLAG8,SFLAG8,VALUE9,MFLAG9,QFLAG9,SFLAG9,VALUE10,MFLAG10, | |
| QFLAG10,SFLAG10,VALUE11,MFLAG11,QFLAG11,SFLAG11,VALUE12,MFLAG12,QFLAG12,SFLAG12,VALUE13,MFLAG13,QFLAG13,SFLAG13,VALUE14,MFLAG14,QFLAG14,SFLAG14, | |
| VALUE15,MFLAG15,QFLAG15,SFLAG15,VALUE16,MFLAG16,QFLAG16,SFLAG16,VALUE17,MFLAG17,QFLAG17,SFLAG17,VALUE18,MFLAG18,QFLAG18,SFLAG18,VALUE19,MFLAG19, | |
| QFLAG19,SFLAG19,VALUE20,MFLAG20,QFLAG20,SFLAG20,VALUE21,MFLAG21,QFLAG21,SFLAG21,VALUE22,MFLAG22,QFLAG22,SFLAG22,VALUE23,MFLAG23,QFLAG23,SFLAG23, | |
| VALUE24,MFLAG24,QFLAG24,SFLAG24,VALUE25,MFLAG25,QFLAG25,SFLAG25,VALUE26,MFLAG26,QFLAG26,SFLAG26,VALUE27,MFLAG27,QFLAG27,SFLAG27,VALUE28,MFLAG28, | |
| QFLAG28,SFLAG28,VALUE29,MFLAG29,QFLAG29,SFLAG29,VALUE30,MFLAG30,QFLAG30,SFLAG30,VALUE31,MFLAG31,QFLAG31,SFLAG31)) | |
| # rename data frame | |
| rename!(out_df, colnames) | |
| # data statistics | |
| # subset TMAX/TMIN/TAVG Elements | |
| tmax = filter(:ELEMENT => isequal("TMAX"), out_df) | |
| tmin = filter(:ELEMENT => isequal("TMIN"), out_df) | |
| tavg = filter(:ELEMENT => isequal("TAVG"), out_df) | |
| month_index = vcat(1,2,3,4,collect(5:4:128)) # print(vcat(1,2,3,4,collect(5:4:128))) | |
| tmax_mo = tmax[:,[1, 2, 3, 4, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125]] | |
| tmin_mo = tmin[:,[1, 2, 3, 4, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125]] | |
| tavg_mo = tavg[:,[1, 2, 3, 4, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125]] | |
| # build date index for the set | |
| days_total = 31 | |
| #day_index = repeat(vcat(["01","02","03","04","05","06","07","08","09"],string.(collect(10:1:31))),inner = size(tmax,1)) | |
| day_index = vcat(["01","02","03","04","05","06","07","08","09"],string.(collect(10:1:days_total))) | |
| # Loop through creating date index for the data and pull the TMAX temperature data | |
| dims_tmax = size(tmax,1) * days_total | |
| out_tmax = fill("", dims_tmax) # push to | |
| data_out_tmax = zeros(dims_tmax) # push to | |
| data_grab = [] # temp for loop | |
| temp = [] # temp for loop | |
| count = 0 | |
| count1 = 0 | |
| for j = 5:size(tmax_mo,2) # loop along columns | |
| #print("this is fucking JJJ",j,"yee\n") | |
| count = count +1 | |
| for i = 1 :size(tmax_mo,1) # loop down the rows | |
| count1 = count1+1 | |
| if tmax_mo[i,3] == "02" && (day_index[count] == "29" || day_index[count] == "30" || day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmax_mo[i,3] == "04" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmax_mo[i,3] == "06" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmax_mo[i,3] == "09" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmax_mo[i,3] == "11" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmax_mo[i,3] != "01" || tmax_mo[i,3] != "04" || tmax_mo[i,3] != "06" || tmax_mo[i,3] != "09" || tmax_mo[i,3] != "11" | |
| temp = join([tmax_mo[i,2],tmax_mo[i,3],day_index[count]]) | |
| data_grab = tmax_mo[i,j] | |
| end | |
| #push!(out_tmax,temp) | |
| #push!(data_out_tmax,data_grab) | |
| out_tmax[count1] = temp | |
| data_out_tmax[count1] = data_grab | |
| end | |
| end | |
| dims_tmin = size(tmin,1) * days_total | |
| out_tmin = fill("", dims_tmin) # push to | |
| data_out_tmin = zeros(dims_tmin) # push to | |
| data_grab = [] # temp for loop | |
| temp = [] # temp for loop | |
| count = 0 | |
| count1 = 0 | |
| for j = 5:size(tmin_mo,2) # loop along columns | |
| #print("this is fucking JJJ",j,"yee\n") | |
| count = count +1 | |
| for i = 1 :size(tmin_mo,1) # loop down the rows | |
| count1 = count1+1 | |
| if tmin_mo[i,3] == "02" && (day_index[count] == "29" || day_index[count] == "30" || day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmin_mo[i,3] == "04" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmin_mo[i,3] == "06" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmin_mo[i,3] == "09" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmin_mo[i,3] == "11" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tmin_mo[i,3] != "01" || tmin_mo[i,3] != "04" || tmin_mo[i,3] != "06" || tmin_mo[i,3] != "09" || tmin_mo[i,3] != "11" | |
| temp = join([tmin_mo[i,2],tmin_mo[i,3],day_index[count]]) | |
| data_grab = tmin_mo[i,j] | |
| end | |
| out_tmin[count1] = temp | |
| data_out_tmin[count1] = data_grab | |
| end | |
| end | |
| dims_avg = size(tavg,1) * days_total | |
| if dims_avg != 0 | |
| out_tavg = fill("", dims_avg) # push to | |
| data_out_tavg = zeros(dims_avg) # push to | |
| data_grab = [] # temp for loop | |
| temp = [] # temp for loop | |
| count = 0 | |
| count1 = 0 | |
| for j = 5:size(tavg_mo,2) # loop along columns | |
| #print("this is fucking JJJ",j,"yee\n") | |
| count = count +1 | |
| for i = 1 :size(tavg_mo,1) # loop down the rows | |
| count1 = count1+1 | |
| if tavg_mo[i,3] == "02" && (day_index[count] == "29" || day_index[count] == "30" || day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tavg_mo[i,3] == "04" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tavg_mo[i,3] == "06" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tavg_mo[i,3] == "09" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tavg_mo[i,3] == "11" && (day_index[count] == "31") | |
| temp = "" | |
| data_grab = 99999.999 | |
| elseif tavg_mo[i,3] != "01" || tavg_mo[i,3] != "04" || tavg_mo[i,3] != "06" || tavg_mo[i,3] != "09" || tavg_mo[i,3] != "11" | |
| temp = join([tavg_mo[i,2],tavg_mo[i,3],day_index[count]]) | |
| data_grab = tavg_mo[i,j] | |
| end | |
| out_tavg[count1] = temp | |
| data_out_tavg[count1] = data_grab | |
| end | |
| end | |
| end | |
| # subset to remove missing data for the shorter months | |
| out_tmax = out_tmax[out_tmax .!= ""] # | |
| data_out_tmax = data_out_tmax[data_out_tmax .!= 99999.999] | |
| out_tmin = out_tmin[out_tmin .!= ""] # | |
| data_out_tmin = Float64.(data_out_tmin[data_out_tmin .!= 99999.999]) | |
| if dims_avg != 0 | |
| out_tavg = out_tavg[out_tavg .!= ""] # | |
| data_out_tavg = Float64.(data_out_tavg[data_out_tavg .!= 99999.999]) | |
| end | |
| # convert date to date format | |
| out_tmax = Date.(out_tmax, "yyyymmdd") | |
| out_tmin = Date.(out_tmin, "yyyymmdd") | |
| if dims_avg != 0 | |
| out_tavg = Date.(out_tavg, "yyyymmdd") | |
| end | |
| # create final ordered by date dataframe | |
| out_tmax = DataFrame(Date = out_tmax, TMAX = data_out_tmax) | |
| final_tmax = sort(out_tmax,[:Date]) | |
| out_tmin = DataFrame(Date = out_tmin, TMIN = data_out_tmin) | |
| final_tmin = sort(out_tmin,[:Date]) | |
| if dims_avg != 0 | |
| out_tavg = DataFrame(Date = out_tavg, TAVG = data_out_tavg) | |
| final_tavg = sort(out_tavg,[:Date]) | |
| end | |
| ####################################################### | |
| # left join to date index | |
| ####################################################### | |
| #=test_date = final_tmax[1:5,1] | |
| dummy_temp = [1.0,1.0,1.0,1.0,1.0] | |
| df1 = DataFrame(Date = test_date, TMAX = dummy_temp) | |
| global date_index_tmax = leftjoin(date_index_tmax, final_tmax, on = :Date, makeunique=true) | |
| global date_index_tmin = leftjoin(date_index_tmin, final_tmin, on = :Date, makeunique=true) | |
| =# | |
| # Build output data frames (all parsed from fixed width .dly file) | |
| #global all_out_tmax = vcat(all_out_tmax,final_tmax) | |
| #global all_out_tmin = vcat(all_out_tmin,final_tmin) | |
| #global all_out_tavg = vcat(all_out_tavg,final_tavg) | |
| # write parsed .csv files to file | |
| CSV.write(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all_csv/tmax/",all_dly_files_for_export[d],".csv"]),final_tmax) | |
| CSV.write(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all_csv/tmin/",all_dly_files_for_export[d],".csv"]),final_tmin) | |
| if dims_avg != 0 | |
| CSV.write(join(["C:/Users/andrew.bannerman/Desktop/Julia/scripts/GHCN data/ghcnd_all_csv/tavg/",all_dly_files_for_export[d],".csv"]),final_tavg) | |
| end | |
| end # loop |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment