Skip to content

Commit 01a2bb2

Browse files
abarciauskas-bgseabarciauskas-bgse
committed
Add CMIP6 demo
1 parent 1ad2481 commit 01a2bb2

File tree

4 files changed

+384
-0
lines changed

4 files changed

+384
-0
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,6 @@ cdk.out/
106106
node_modules/
107107

108108
.pgdata
109+
110+
# Demo files to ignore
111+
demo/cmip6/CMIP6_daily_*stac_items.ndjson

‎demo/cmip6/CMIP6_daily_GISS-E2-1-G_tas_collection.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"id": "CMIP6_daily_GISS-E2-1-G_tas",
3+
"stac_version": "1.0.0",
4+
"license": "not-provided",
5+
"title": "CMIP6 Daily Near-Surface Air Temperature from GISS-E2-1-G Model",
6+
"type": "Collection",
7+
"description": "CMIP6 Daily Near-Surface Air Temperature from GISS-E2-1-G Model",
8+
"links":[],
9+
"extent": {
10+
"spatial": {
11+
"bbox": [
12+
[
13+
-180,
14+
-90,
15+
180,
16+
90
17+
]
18+
]
19+
},
20+
"temporal": {
21+
"interval": [
22+
[
23+
"1950-01-01T00:00:00.000Z",
24+
"1951-12-31T00:00:00.000Z"
25+
]
26+
]
27+
}
28+
}
29+
}

‎demo/cmip6/generate_cmip6_items.ipynb‎

Lines changed: 332 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,332 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "9de99346-8c2a-48bb-b79b-e28a4db099d8",
6+
"metadata": {},
7+
"source": [
8+
"# Generate CMIP STAC Items and Load them into a pgSTAC database\n",
9+
"\n",
10+
"This notebook walks through generating STAC items from [NEX GDDP CMIP6 COGs on AWS](https://aws.amazon.com/marketplace/pp/prodview-k6adk576fiwmm#resources).\n",
11+
"\n",
12+
"As-is it uses daily data from the `GISS-E2-1-G` model, the `tas` variable and loads data from 1950 and 1951. The bucket has other data available. It includes monthly aggregates, other models, other variables and more years. The scripts below can easily be modified to STAC-ify other data in the nex-gddp-cmip6-cog bucket."
13+
]
14+
},
15+
{
16+
"cell_type": "code",
17+
"execution_count": 1,
18+
"id": "6f788363",
19+
"metadata": {},
20+
"outputs": [],
21+
"source": [
22+
"import boto3\n",
23+
"import fsspec\n",
24+
"import json\n",
25+
"from pystac import Catalog, Collection, Item, Asset, MediaType\n",
26+
"from datetime import datetime\n",
27+
"import rio_stac\n",
28+
"from pprint import pprint\n",
29+
"import concurrent.futures\n",
30+
"import threading"
31+
]
32+
},
33+
{
34+
"cell_type": "code",
35+
"execution_count": 2,
36+
"id": "8e4cfbb8",
37+
"metadata": {
38+
"tags": [
39+
"parameters"
40+
]
41+
},
42+
"outputs": [],
43+
"source": [
44+
"# Specify the CMIP collection to use (daily or monthly)\n",
45+
"model = \"GISS-E2-1-G\"\n",
46+
"variable = \"tas\""
47+
]
48+
},
49+
{
50+
"cell_type": "code",
51+
"execution_count": 3,
52+
"id": "cdaccd78",
53+
"metadata": {},
54+
"outputs": [],
55+
"source": [
56+
"anon = True\n",
57+
"s3_path = f\"s3://nex-gddp-cmip6-cog/daily/{model}/historical/r1i1p1f2/{variable}/\""
58+
]
59+
},
60+
{
61+
"cell_type": "code",
62+
"execution_count": 4,
63+
"id": "a7caab29",
64+
"metadata": {},
65+
"outputs": [],
66+
"source": [
67+
"fs_read = fsspec.filesystem(\"s3\", anon=anon)"
68+
]
69+
},
70+
{
71+
"cell_type": "code",
72+
"execution_count": 5,
73+
"id": "4936f757",
74+
"metadata": {},
75+
"outputs": [
76+
{
77+
"name": "stdout",
78+
"output_type": "stream",
79+
"text": [
80+
"23725 discovered from s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/\n"
81+
]
82+
}
83+
],
84+
"source": [
85+
"file_paths = fs_read.glob(f\"{s3_path}*\")\n",
86+
"print(f\"{len(file_paths)} discovered from {s3_path}\")"
87+
]
88+
},
89+
{
90+
"cell_type": "code",
91+
"execution_count": 6,
92+
"id": "999b0670",
93+
"metadata": {},
94+
"outputs": [],
95+
"source": [
96+
"# Here we prepend the prefix 's3://', which points to AWS.\n",
97+
"subset_files = sorted([\"s3://\" + f for f in file_paths if \"_1950_\" in f or \"_1951_\" in f])"
98+
]
99+
},
100+
{
101+
"cell_type": "code",
102+
"execution_count": 7,
103+
"id": "6af269dc",
104+
"metadata": {},
105+
"outputs": [
106+
{
107+
"name": "stdout",
108+
"output_type": "stream",
109+
"text": [
110+
"Subseted data to files for 1950 and 1951. 730 files to process.\n"
111+
]
112+
}
113+
],
114+
"source": [
115+
"if len(subset_files) == 0:\n",
116+
" raise Exception(f\"No files to process. Do COGs for the {model} model exist?\")\n",
117+
"else:\n",
118+
" print(f\"Subseted data to files for 1950 and 1951. {len(subset_files)} files to process.\")"
119+
]
120+
},
121+
{
122+
"cell_type": "code",
123+
"execution_count": 8,
124+
"id": "57dc1b5f",
125+
"metadata": {},
126+
"outputs": [],
127+
"source": [
128+
"file_prefix = f\"CMIP6_daily_{model}_{variable}\"\n",
129+
"stac_items_file = f\"{file_prefix}_stac_items.ndjson\"\n",
130+
"collection_json = json.loads(open(f'{file_prefix}_collection.json').read())\n",
131+
"collection = Collection.from_dict(collection_json)"
132+
]
133+
},
134+
{
135+
"cell_type": "code",
136+
"execution_count": 9,
137+
"id": "45771e88",
138+
"metadata": {
139+
"lines_to_next_cell": 1
140+
},
141+
"outputs": [],
142+
"source": [
143+
"# clear the file\n",
144+
"with open(stac_items_file, 'w') as file:\n",
145+
" pass"
146+
]
147+
},
148+
{
149+
"cell_type": "code",
150+
"execution_count": 10,
151+
"id": "ececf9d5",
152+
"metadata": {},
153+
"outputs": [],
154+
"source": [
155+
"def process_item(s3_file, file, lock):\n",
156+
" print(f\"Processing {s3_file}\")\n",
157+
" filename = s3_file.split('/')[-1]\n",
158+
" year, month, day = filename.split('_')[-3:]\n",
159+
" day = day.replace('.tif', '')\n",
160+
" datetime_ = datetime.strptime(f'{year}{month}{day}', '%Y%m%d') \n",
161+
" # Create a new Item\n",
162+
" item = rio_stac.create_stac_item(\n",
163+
" id=filename,\n",
164+
" source=s3_file,\n",
165+
" collection=collection.id,\n",
166+
" input_datetime=datetime_,\n",
167+
" with_proj=True,\n",
168+
" with_raster=True,\n",
169+
" asset_name=\"data\",\n",
170+
" asset_roles=[\"data\"],\n",
171+
" asset_media_type=\"image/tiff; application=geotiff; profile=cloud-optimized\"\n",
172+
" )\n",
173+
" tiling_asset = Asset(\n",
174+
" href=s3_file,\n",
175+
" roles=['virtual', 'tiling'],\n",
176+
" title='tiling',\n",
177+
" description='Virtual asset for tiling',\n",
178+
" extra_fields={\n",
179+
" 'compose:rescale': [210, 330],\n",
180+
" 'compose:colormap_name': 'hot'\n",
181+
" }\n",
182+
" )\n",
183+
" item.assets['tiling'] = tiling_asset\n",
184+
" with lock:\n",
185+
" file.write(json.dumps(item.to_dict()) + '\\n')"
186+
]
187+
},
188+
{
189+
"cell_type": "code",
190+
"execution_count": null,
191+
"id": "645d3ccb",
192+
"metadata": {},
193+
"outputs": [
194+
{
195+
"name": "stdout",
196+
"output_type": "stream",
197+
"text": [
198+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_01.tif\n",
199+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_02.tif\n",
200+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_03.tif\n",
201+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_04.tif\n",
202+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_05.tif\n",
203+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_06.tif\n",
204+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_07.tif\n",
205+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_08.tif\n",
206+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_09.tif\n",
207+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_10.tif\n",
208+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_11.tif\n",
209+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_12.tif\n",
210+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_13.tifProcessing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_14.tif\n",
211+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_15.tif\n",
212+
"\n",
213+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_16.tif\n",
214+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_17.tif\n",
215+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_18.tif\n",
216+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_19.tif\n",
217+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_20.tif\n",
218+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_21.tif\n",
219+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_22.tif\n",
220+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_23.tif\n",
221+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_24.tif\n",
222+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_25.tif\n",
223+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_26.tif\n",
224+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_27.tif\n",
225+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_28.tif\n",
226+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_29.tif\n",
227+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_30.tif\n",
228+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_31.tif\n",
229+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_01.tif\n",
230+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_02.tif\n",
231+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_03.tif\n",
232+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_04.tif\n",
233+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_05.tif\n",
234+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_06.tif\n",
235+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_07.tif\n",
236+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_08.tif\n",
237+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_09.tif\n",
238+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_10.tif\n",
239+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_11.tif\n",
240+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_12.tif\n",
241+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_13.tif\n",
242+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_14.tif\n",
243+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_15.tif\n",
244+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_16.tif\n",
245+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_17.tif\n",
246+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_18.tif\n",
247+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_19.tif\n",
248+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_20.tif\n",
249+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_21.tif\n",
250+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_22.tif\n",
251+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_23.tif\n",
252+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_24.tif\n",
253+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_25.tif\n",
254+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_26.tif\n",
255+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_27.tif\n",
256+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_02_28.tif\n",
257+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_01.tif\n",
258+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_02.tif\n",
259+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_03.tif\n",
260+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_04.tif\n",
261+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_05.tif\n",
262+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_06.tif\n",
263+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_07.tif\n",
264+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_08.tif\n",
265+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_09.tif\n",
266+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_10.tif\n",
267+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_11.tif\n",
268+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_12.tif\n",
269+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_13.tif\n",
270+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_14.tif\n",
271+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_15.tif\n",
272+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_16.tif\n",
273+
"Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_03_17.tif\n"
274+
]
275+
}
276+
],
277+
"source": [
278+
"lock = threading.Lock()\n",
279+
"file = open(stac_items_file, 'a')\n",
280+
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
281+
" futures = [executor.submit(process_item, obj, file, lock) for obj in subset_files]\n",
282+
" [future.result() for future in concurrent.futures.as_completed(futures)]\n",
283+
"file.close()"
284+
]
285+
},
286+
{
287+
"cell_type": "code",
288+
"execution_count": 21,
289+
"id": "d1728b26-9f8f-4bef-8704-a37fe9af2f9f",
290+
"metadata": {},
291+
"outputs": [
292+
{
293+
"name": "stdout",
294+
"output_type": "stream",
295+
"text": [
296+
"Inserting collection from CMIP6_daily_GISS-E2-1-G_tas_collection.json\n",
297+
"Inserting items from CMIP6_daily_GISS-E2-1-G_tas_stac_items.ndjson\n"
298+
]
299+
}
300+
],
301+
"source": [
302+
"!./seed-db.sh {model} {variable}"
303+
]
304+
}
305+
],
306+
"metadata": {
307+
"jupytext": {
308+
"cell_metadata_filter": "-all",
309+
"formats": "ipynb,py",
310+
"main_language": "python"
311+
},
312+
"kernelspec": {
313+
"display_name": "Python 3 (ipykernel)",
314+
"language": "python",
315+
"name": "python3"
316+
},
317+
"language_info": {
318+
"codemirror_mode": {
319+
"name": "ipython",
320+
"version": 3
321+
},
322+
"file_extension": ".py",
323+
"mimetype": "text/x-python",
324+
"name": "python",
325+
"nbconvert_exporter": "python",
326+
"pygments_lexer": "ipython3",
327+
"version": "3.10.12"
328+
}
329+
},
330+
"nbformat": 4,
331+
"nbformat_minor": 5
332+
}

‎demo/cmip6/seed-db.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
model="$1"
3+
variable="$2"
4+
5+
collection_json_file="CMIP6_daily_${model}_${variable}_collection.json"
6+
items_json_file="CMIP6_daily_${model}_${variable}_stac_items.ndjson"
7+
8+
if [ -z "$DATABASE_URL" ]; then
9+
username=postgres
10+
password=password
11+
host=localhost
12+
dbname=postgres
13+
port=5432
14+
DATABASE_URL="postgresql://$username:$password@$host:$port/$dbname"
15+
fi
16+
17+
echo "Inserting collection from $collection_json_file"
18+
pypgstac load collections $collection_json_file --dsn $DATABASE_URL --method insert_ignore
19+
echo "Inserting items from $items_json_file"
20+
pypgstac load items $items_json_file --dsn $DATABASE_URL --method insert_ignore

0 commit comments

Comments
 (0)