diff --git a/Project.toml b/Project.toml index da1f611..0599047 100644 --- a/Project.toml +++ b/Project.toml @@ -1,29 +1,25 @@ name = "SentinelDataSource" uuid = "afd1bdb2-e5b9-463c-8ecd-ef9b9458f1b0" -authors = ["Felix Cremer and contributors"] version = "0.1.0-DEV" +authors = ["Felix Cremer and contributors"] [deps] CFTime = "179af706-886a-5703-950a-314cd64e0468" CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157" DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" -DiskArrayTools = "fcd2136c-9f69-4db6-97e5-f31981721d63" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" -YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" -YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" +TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" ZarrDatasets = "519a4cdf-1362-424a-9ea1-b1d782dbb24b" [compat] CFTime = "0.2.1" -CommonDataModel = "0.3.8" +CommonDataModel = "0.3.8, 0.4" DimensionalData = "0.29.12" -DiskArrayTools = "0.1.12" DiskArrays = "0.4.11" Rasters = "0.14.4" -YAXArrayBase = "0.7.5" -YAXArrays = "0.6.1" +TimerOutputs = "0.5.29" Zarr = "0.9.4" ZarrDatasets = "0.1.3" julia = "1.6.7" diff --git a/docs/src/spectraldiversity.jl b/docs/src/spectraldiversity.jl new file mode 100644 index 0000000..b366a38 --- /dev/null +++ b/docs/src/spectraldiversity.jl @@ -0,0 +1,47 @@ +using SentinelDataSource +using YAXArrays +using Distances +using DimensionalData +using AWS + +struct AnonymousGCS <:AbstractAWSConfig end +struct NoCredentials end +AWS.region(aws::AnonymousGCS) = "" # No region +AWS.credentials(aws::AnonymousGCS) = NoCredentials() # No credentials +AWS.check_credentials(c::NoCredentials) = c # Skip credentials check +AWS.sign!(aws::AnonymousGCS, ::AWS.Request) = nothing # Don't sign request +function AWS.generate_service_url(aws::AnonymousGCS, service::String, resource::String) + service == "s3" || throw(ArgumentError("Can only handle s3 requests to GCS")) + awsurl = string("https://objects.eodc.eu:2222", resource) + return awsurl +end +AWS.global_aws_config(AnonymousGCS()) +path = "s3://e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2A_MSIL2A_20180601T102021_N0500_R065_T32UPC_20230902T045008.zarr/measurements/reflectance/r20m" + +pathbase = "s3://e05ab01a9d56408d82ac32d69a5aae2a:sample-data/eopf-sample-output/" +zopen(path) +c = Cube(open_dataset(zopen(path))) +yax = YAXArray((Dim{:x}(1:500), Dim{:y}(500:-1:1), Dim{:Variables}(1:10)), rand(500,500,10)) +c = yax +function spectraldiversity(c) + dlat = dims(c, :x) + dlon = dims(c, :y) + dband = dims(c, :Variables) + # This needs a better interface for within index space + latinterval = MovingIntervals(center=dlat.val, width=3*step(dlat), n=length(dlat)-1, step=step(dlat)) + loninterval = MovingIntervals(center=dlon.val, width=-3*step(dlon), n=length(dlon)-1, step=-step(dlon)) + bandinterval = MovingIntervals(:closed, :closed, left=first(dband.val), right=last(dband.val), n=1) + meanwindows = windows(c, :y=>loninterval, :Variables => Whole()) + @show meanwindows[10,10,1] + @time xmap(innerdiversity, meanwindows) +end +spectraldiversity(c) +function innerdiversity(xout, arr) + winsize = prod(size(arr)) รท size(arr,3) + arrreshaped = reshape(arr, (winsize, size(arr,3))) + tslist = eachrow(arrreshaped) + dists = pairwise(Euclidean(), tslist) .* 2 ./ winsize .^ 4 + xout .= sum(dists) ./ 2 +end + +spectraldiversity(c) \ No newline at end of file diff --git a/src/SentinelDataSource.jl b/src/SentinelDataSource.jl index 34618e2..6720b0e 100644 --- a/src/SentinelDataSource.jl +++ b/src/SentinelDataSource.jl @@ -1,24 +1,38 @@ module SentinelDataSource -using DimensionalData: DimTree +using DimensionalData: DimTree, DimArray, DimensionalData using Zarr: zopen using ZarrDatasets: ZarrDataset using CommonDataModel: CommonDataModel as CDM -using Rasters:Raster - +using Rasters: Rasters +using TimerOutputs export open_tree open_tree(path::AbstractString) = open_tree(ZarrDataset(path)) function open_tree(dataset::ZarrDataset) - stem = DimTree() - groupnames = CDM.groupnames(dataset) - varnames = CDM.varnames(dataset) - alldimnames = nesteddimnames(dataset) - for v in setdiff(varnames, alldimnames) - setindex!(stem, Raster(CDM.variable(dataset, v), lazy=true),Symbol(v)) + @timeit_debug "stem" stem = DimTree() + @timeit_debug "groups" groupnames = CDM.groupnames(dataset) + @timeit_debug "vars" varnames = CDM.varnames(dataset) + @timeit_debug "dims" alldimnames = nesteddimnames(dataset) + diffnames = setdiff(varnames, alldimnames) + for v in diffnames + @timeit_debug "var $v" begin + var = CDM.variable(dataset, v) + vardims = Rasters._dims(var) + metadata_out = Rasters._metadata(var) + missingval_out = Rasters._read_missingval_pair(var, metadata_out, Rasters.nokw) + mod = Rasters._mod(eltype(var), metadata_out, missingval_out;scaled=true, coerce=true) + #= + Rasters.FileArray{ZarrDataset}(var, filename; + name=v, Rasters.nokw, mod, write=false + ) + =# + vardata = Rasters._maybe_modify(var, mod) + setindex!(stem, DimArray(vardata, vardims),Symbol(v)) + end end for g in groupnames - setindex!(stem, open_tree(CDM.group(dataset, g)),Symbol(g)) + @timeit_debug "forg $g" setindex!(stem, open_tree(CDM.group(dataset, g)),Symbol(g)) end stem end