{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Reconstruction using GraphEM" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Expected time to run through: 5 hrs**\n", "\n", "This tutorial demonstrates how to get a reconstruction using GraphEM, leveraging HadCRUT4 and PAGES2k." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test data preparation\n", "\n", "To go through this tutorial, please prepare test data following the steps:\n", "\n", "Download the test case named “PAGES2k_HadCRUT” with this [link](https://drive.google.com/drive/folders/12V0Ny4eO2HKU40-Iy5Jiw2SAfVNzXXcd?usp=sharing).\n", "Create a directory named “testcases” in the same directory where this notebook sits.\n", "Put the unzipped direcotry “PAGES2k_HadCRUT” into “testcases”.\n", "\n", "Below, we first load some useful packages, including our `GraphEM`." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", "\n", "import LMRt\n", "import GraphEM\n", "import os\n", "import numpy as np\n", "import pandas as pd\n", "import xarray as xr\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Low-level workflow" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "job = GraphEM.ReconJob()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m\u001b[36mGraphEM: job.load_configs() >>> loading reconstruction configurations from: ./testcases/PAGES2k_HadCRUT/configs.yml\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.load_configs() >>> job.configs created\u001b[0m\n", "\u001b[1m\u001b[36mGraphEM: job.load_configs() >>> job.configs[\"job_dirpath\"] = /Users/fzhu/Github/GraphEM/docsrc/tutorial/testcases/PAGES2k_HadCRUT/recon\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.load_configs() >>> /Users/fzhu/Github/GraphEM/docsrc/tutorial/testcases/PAGES2k_HadCRUT/recon created\u001b[0m\n", "{'anom_period': [1951, 1980],\n", " 'calib_period': [1930, 2000],\n", " 'job_dirpath': '/Users/fzhu/Github/GraphEM/docsrc/tutorial/testcases/PAGES2k_HadCRUT/recon',\n", " 'job_id': 'GraphEM_tutorial',\n", " 'obs_path': {'tas': './data/obs/HadCRUT.5.0.1.0.analysis.anomalies.ensemble_mean.nc'},\n", " 'obs_varname': {'lat': 'latitude', 'lon': 'longitude', 'tas': 'tas_mean'},\n", " 'proxydb_path': './data/proxy/pages2k_dataset.pkl',\n", " 'ptype_list': ['coral.d18O',\n", " 'coral.SrCa',\n", " 'coral.calc',\n", " 'tree.TRW',\n", " 'tree.MXD'],\n", " 'recon_period': [1500, 2000]}\n" ] } ], "source": [ "job.load_configs('./testcases/PAGES2k_HadCRUT/configs.yml', verbose=True)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m\u001b[36mGraphEM: job.load_proxydb() >>> job.configs[\"proxydb_path\"] = /Users/fzhu/Github/GraphEM/docsrc/tutorial/testcases/PAGES2k_HadCRUT/data/proxy/pages2k_dataset.pkl\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.load_proxydb() >>> 692 records loaded\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.load_proxydb() >>> job.proxydb created\u001b[0m\n" ] } ], "source": [ "job.load_proxydb(verbose=True)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m\u001b[36mGraphEM: job.filter_proxydb() >>> filtering proxy records according to: ['coral.d18O', 'coral.SrCa', 'coral.calc', 'tree.TRW', 'tree.MXD']\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.filter_proxydb() >>> 519 records remaining\u001b[0m\n" ] } ], "source": [ "job.filter_proxydb(verbose=True)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m\u001b[36mGraphEM: job.seasonalize_proxydb() >>> job.configs[\"ptype_season\"] = {'coral.d18O': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'coral.SrCa': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'coral.calc': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'tree.TRW': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'tree.MXD': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]}\u001b[0m\n", "\u001b[1m\u001b[36mGraphEM: job.seasonalize_proxydb() >>> seasonalizing proxy records according to: {'coral.d18O': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'coral.SrCa': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'coral.calc': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'tree.TRW': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'tree.MXD': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]}\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.seasonalize_proxydb() >>> 519 records remaining\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.seasonalize_proxydb() >>> job.proxydb updated\u001b[0m\n" ] } ], "source": [ "# ptype_season = {}\n", "# for k, v in job.proxydb.type_dict.items():\n", "# ptype_season[k] = list(range(1, 13)) # annual\n", " \n", "# job.seasonalize_proxydb(ptype_season, verbose=True)\n", "job.seasonalize_proxydb(verbose=True)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m\u001b[36mGraphEM: job.load_obs() >>> loading instrumental observation fields from: {'tas': '/Users/fzhu/Github/GraphEM/docsrc/tutorial/testcases/PAGES2k_HadCRUT/data/obs/HadCRUT.5.0.1.0.analysis.anomalies.ensemble_mean.nc'}\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.load_obs() >>> job.obs created\u001b[0m\n" ] } ], "source": [ "job.load_obs(varname_dict={'lat': 'latitude', 'lon': 'longitude', 'tas': 'tas_mean'}, verbose=True)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m\u001b[36mGraphEM: job.seasonalize_obs() >>> job.configs[\"obs_season\"] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]\u001b[0m\n", "\u001b[1m\u001b[30mGraphEM: job.seasonalize_obs() >>> seasonalized obs w/ season [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]\u001b[0m\n", "Dataset Overview\n", "-----------------------\n", "\n", " Name: tas\n", " Source: /Users/fzhu/Github/GraphEM/docsrc/tutorial/testcases/PAGES2k_HadCRUT/data/obs/HadCRUT.5.0.1.0.analysis.anomalies.ensemble_mean.nc\n", " Shape: time:171, lat:36, lon:72\n", "\u001b[1m\u001b[32mGraphEM: job.seasonalize_obs() >>> job.obs updated\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/Users/fzhu/Github/LMRt/LMRt/utils.py:261: RuntimeWarning: Mean of empty slice\n", " tmp = np.nanmean(var[inds, ...], axis=0)\n" ] } ], "source": [ "job.seasonalize_obs(verbose=True)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1m\u001b[32mGraphEM: job.prep_data() >>> job.recon_time created\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.prep_data() >>> job.calib_time created\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.prep_data() >>> job.calib_idx created\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.prep_data() >>> job.temp created\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.prep_data() >>> job.df_proxy created\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.prep_data() >>> job.proxy created\u001b[0m\n", "\u001b[1m\u001b[32mGraphEM: job.prep_data() >>> job.lonlat created\u001b[0m\n" ] } ], "source": [ "job.prep_data(verbose=True)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | NAm_153 | \n", "NAm_165 | \n", "Asi_178 | \n", "Asi_174 | \n", "Asi_198 | \n", "NAm_145 | \n", "Arc_071 | \n", "NAm_102 | \n", "NAm_046 | \n", "Ocn_065 | \n", "... | \n", "NAm_143 | \n", "Asi_208 | \n", "Asi_119 | \n", "Ocn_153 | \n", "Asi_026 | \n", "Ocn_169 | \n", "Asi_201 | \n", "Asi_179 | \n", "Ocn_071 | \n", "Ocn_072 | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
1500.0 | \n", "NaN | \n", "NaN | \n", "0.994 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "1.06 | \n", "NaN | \n", "1.026 | \n", "NaN | \n", "... | \n", "NaN | \n", "0.805 | \n", "0.849 | \n", "NaN | \n", "0.710 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
1501.0 | \n", "NaN | \n", "NaN | \n", "1.131 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "0.70 | \n", "NaN | \n", "1.058 | \n", "NaN | \n", "... | \n", "NaN | \n", "0.694 | \n", "0.882 | \n", "NaN | \n", "0.759 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
1502.0 | \n", "NaN | \n", "NaN | \n", "0.898 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "1.31 | \n", "NaN | \n", "1.088 | \n", "NaN | \n", "... | \n", "NaN | \n", "0.742 | \n", "0.620 | \n", "NaN | \n", "0.944 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
1503.0 | \n", "NaN | \n", "NaN | \n", "1.032 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "0.70 | \n", "NaN | \n", "0.875 | \n", "NaN | \n", "... | \n", "NaN | \n", "0.677 | \n", "0.413 | \n", "NaN | \n", "0.845 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
1504.0 | \n", "NaN | \n", "NaN | \n", "0.832 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "-0.43 | \n", "NaN | \n", "1.139 | \n", "NaN | \n", "... | \n", "NaN | \n", "0.730 | \n", "0.551 | \n", "NaN | \n", "0.702 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
1996.0 | \n", "1.346 | \n", "NaN | \n", "1.028 | \n", "NaN | \n", "1.647 | \n", "1.146 | \n", "1.36 | \n", "1.033 | \n", "1.373 | \n", "NaN | \n", "... | \n", "NaN | \n", "1.490 | \n", "0.813 | \n", "9.099227 | \n", "0.922 | \n", "NaN | \n", "1.070 | \n", "1.044 | \n", "-4.114561 | \n", "8.819362 | \n", "
1997.0 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "1.751 | \n", "1.263 | \n", "2.30 | \n", "1.037 | \n", "1.153 | \n", "NaN | \n", "... | \n", "NaN | \n", "1.278 | \n", "0.864 | \n", "9.192620 | \n", "1.112 | \n", "NaN | \n", "1.007 | \n", "1.166 | \n", "-4.231963 | \n", "8.767808 | \n", "
1998.0 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "1.499 | \n", "1.054 | \n", "0.67 | \n", "1.171 | \n", "1.369 | \n", "NaN | \n", "... | \n", "NaN | \n", "1.332 | \n", "0.743 | \n", "9.124461 | \n", "1.260 | \n", "NaN | \n", "1.111 | \n", "1.161 | \n", "-4.427678 | \n", "8.751082 | \n", "
1999.0 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "1.054 | \n", "1.067 | \n", "1.47 | \n", "NaN | \n", "1.502 | \n", "NaN | \n", "... | \n", "NaN | \n", "1.161 | \n", "1.269 | \n", "9.073438 | \n", "0.980 | \n", "NaN | \n", "1.402 | \n", "0.851 | \n", "-4.081667 | \n", "8.825333 | \n", "
2000.0 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "0.842 | \n", "0.994 | \n", "1.97 | \n", "NaN | \n", "1.674 | \n", "NaN | \n", "... | \n", "NaN | \n", "0.837 | \n", "1.644 | \n", "9.078696 | \n", "1.152 | \n", "NaN | \n", "0.930 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
501 rows × 519 columns
\n", "