saving and processing

UNFCCC_GHG_data/UNFCCC_reader/Mongolia/config_MNG_BUR2.py

@@ -5,21 +5,21 @@ coords_terminologies = {
 }
 
 inv_conf = {
-    "entity_row" : 0,
-    "unit_row" : 1,
-    "index_cols" : "Greenhouse gas source and sink categories",
-    "header_long" : ["orig_cat_name", "entity", "unit", "time", "data"],
-    "cat_code_regexp" : r"^(?P<code>[a-zA-Z0-9\.]{1,11})[\s\.].*",
-    "cat_codes_manual" : {
-        ' 3.C.1 -Emissions from biomass burning ' : "3.C.1",
-        "Memo Items (5)" : "MEMO",
-        "International Bunkers" : "M.BK",
-        "1.A.3.a.i - International Aviation (International Bunkers) (1)" : "M.BK.A",
-        "1.A.3.d.i - International water-borne navigation (International bunkers) (1)" : "M.BK.M",
-        "1.A.5.c - Multilateral Operations (1)(2)" : "M.MULTIOP",
-        "Total National Emissions and Removals" : "0",
+    "entity_row": 0,
+    "unit_row": 1,
+    "index_cols": "Greenhouse gas source and sink categories",
+    "header_long": ["orig_cat_name", "entity", "unit", "time", "data"],
+    "cat_code_regexp": r"^(?P<code>[a-zA-Z0-9\.]{1,11})[\s\.].*",
+    "cat_codes_manual": {
+        " 3.C.1 -Emissions from biomass burning ": "3.C.1",
+        "Memo Items (5)": "MEMO",
+        "International Bunkers": "M.BK",
+        "1.A.3.a.i - International Aviation (International Bunkers) (1)": "M.BK.A",
+        "1.A.3.d.i - International water-borne navigation (International bunkers) (1)": "M.BK.M",
+        "1.A.5.c - Multilateral Operations (1)(2)": "M.MULTIOP",
+        "Total National Emissions and Removals": "0",
     },
-    "header" : [
+    "header": [
         "Greenhouse gas source and sink categories",
         "CO2",
         "CH4",
@@ -34,7 +34,7 @@ inv_conf = {
         "NMVOCs",
         "SO2",
     ],
-    "unit" : [
+    "unit": [
         "-",
         "Gg",
         "Gg",
@@ -52,84 +52,85 @@ inv_conf = {
 }
 
 inv_conf_per_year = {
-    "1990" : {
-        "pages_to_read" : ["176", "177", "178", "179"],
-        "rows_to_fix" : {
-            3 : ['1.A.2 - Manufacturing Industries and',
-                 "2.B.4 - Caprolactam. Glyoxal and Glyoxylic Acid",
-                 "2.B.8 - Petrochemical and Carbon Black",
-                 "2.D - Non-Energy Products from Fuels and",
-                 '2.F - Product Uses as Substitutes for Ozone',
-                 "3.C - Aggregate sources and non-CO2 emissions",
-                 "3.C.4 - Direct N2O Emissions from managed",
-                 "3.C.5 - Indirect N2O Emissions from managed",
-                 "3.C.6 - Indirect N2O Emissions from manure",
-                 "5.A - Indirect N2O emissions from the atmospheric",
-                 "1.A.3.d.i - International water-borne navigation",
-                 "1.A.3.a.i - International Aviation (International",
-                 ],
-            -2 : ["3.C.1 - Emissions from biomass burning"],
-            2 : [" 3.C.1 -Emissions from biomass burning"],
+    "1990": {
+        "pages_to_read": ["176", "177", "178", "179"],
+        "rows_to_fix": {
+            3: [
+                "1.A.2 - Manufacturing Industries and",
+                "2.B.4 - Caprolactam. Glyoxal and Glyoxylic Acid",
+                "2.B.8 - Petrochemical and Carbon Black",
+                "2.D - Non-Energy Products from Fuels and",
+                "2.F - Product Uses as Substitutes for Ozone",
+                "3.C - Aggregate sources and non-CO2 emissions",
+                "3.C.4 - Direct N2O Emissions from managed",
+                "3.C.5 - Indirect N2O Emissions from managed",
+                "3.C.6 - Indirect N2O Emissions from manure",
+                "5.A - Indirect N2O emissions from the atmospheric",
+                "1.A.3.d.i - International water-borne navigation",
+                "1.A.3.a.i - International Aviation (International",
+            ],
+            -2: ["3.C.1 - Emissions from biomass burning"],
+            2: [" 3.C.1 -Emissions from biomass burning"],
         },
-        'page_defs' : {
-            '176' : {
-                "area" : ['76,501,763,83'],
-                "cols" : ['265,320,360,396,433,471,503,564,624,658,694,741'],
+        "page_defs": {
+            "176": {
+                "area": ["76,501,763,83"],
+                "cols": ["265,320,360,396,433,471,503,564,624,658,694,741"],
             },
-            '177' : {
-                "area" : ['68,542,762,85'],
-                "cols" : ['280,329,374,410,449,482,546,604,637,679,725,751'],
+            "177": {
+                "area": ["68,542,762,85"],
+                "cols": ["280,329,374,410,449,482,546,604,637,679,725,751"],
             },
-            '178' : {
-                "area" : ['71,543,761,81'],
-                "cols" : ['265, 320,361,411,447,483,546,604,621,653,719,746'],
+            "178": {
+                "area": ["71,543,761,81"],
+                "cols": ["265, 320,361,411,447,483,546,604,621,653,719,746"],
             },
-            '179' : {
-                "area" : ['70,542,761,346'],
-                "cols" : ['287,328,365,410,449,482,540,600,636,675,721,750'],
+            "179": {
+                "area": ["70,542,761,346"],
+                "cols": ["287,328,365,410,449,482,540,600,636,675,721,750"],
             },
-        }
+        },
     },
-    "2020" : {
-
-        'page_defs' : {
-            '180' : {
-                "area" : ['70,436,769,86'],
-                "cols" : ['270, 322, 367, 405, 455, 488,550,607,637,669,727,753'],
+    "2020": {
+        "page_defs": {
+            "180": {
+                "area": ["70,436,769,86"],
+                "cols": ["270, 322, 367, 405, 455, 488,550,607,637,669,727,753"],
             },
-            '181' : {
-                "area" : ['68,541,768,86'],
-                "cols" : ['288,343,379,405,460,490,559,600,650,683,729,755'],
+            "181": {
+                "area": ["68,541,768,86"],
+                "cols": ["288,343,379,405,460,490,559,600,650,683,729,755"],
             },
-            '182' : {
-                "area" : ['69, 539, 771, 86'],
-                "cols" : ['273,331,371,425,462,491,560,615,639,671,729,755'],
+            "182": {
+                "area": ["69, 539, 771, 86"],
+                "cols": ["273,331,371,425,462,491,560,615,639,671,729,755"],
             },
-            '183' : {
-                "area" : ['69, 540, 769, 373'],
-                "cols" : ['288, 328,363,425,459,492,560,619,650,683,731,757'],
+            "183": {
+                "area": ["69, 540, 769, 373"],
+                "cols": ["288, 328,363,425,459,492,560,619,650,683,731,757"],
             },
         },
-        "rows_to_fix" : {
-            3 : [
+        "rows_to_fix": {
+            3: [
                 "2.B.8 - Petrochemical and Carbon Black",
                 "2.D - Non-Energy Products from Fuels and",
-                '2.F - Product Uses as Substitutes for Ozone',
+                "2.F - Product Uses as Substitutes for Ozone",
+            ],
+            -2: [
+                "1.C.1 - Transport of CO2",
+                "2.G.2 - SF6 and PFCs from Other Product Uses",
+                "2.G.3 - N2O from Product Uses",
+            ],
+            2: [
+                "3.C - Aggregate sources and non-CO2 emissions",
+                "3.C.4 - Direct N2O Emissions from managed",
+                "3.C.5 - Indirect N2O Emissions from managed",
+                "3.C.6 - Indirect N2O Emissions from manure",
+                "5.A - Indirect N2O emissions from the atmospheric",
+                "1.A.3.d.i - International water-borne navigation",
+                "1.A.3.a.i - International Aviation (International",
+                "2.B.4 - Caprolactam. Glyoxal and Glyoxylic Acid",
             ],
-            -2 : ['1.C.1 - Transport of CO2',
-                  '2.G.2 - SF6 and PFCs from Other Product Uses',
-                  '2.G.3 - N2O from Product Uses',
-
-                  ],
-            2 : ["3.C - Aggregate sources and non-CO2 emissions",
-                 '3.C.4 - Direct N2O Emissions from managed',
-                 "3.C.5 - Indirect N2O Emissions from managed",
-                 "3.C.6 - Indirect N2O Emissions from manure",
-                 '5.A - Indirect N2O emissions from the atmospheric',
-                 "1.A.3.d.i - International water-borne navigation",
-                 "1.A.3.a.i - International Aviation (International",
-                 '2.B.4 - Caprolactam. Glyoxal and Glyoxylic Acid',
-                 ],
         },
     },
 }
@@ -163,7 +164,7 @@ coords_value_mapping = {
         "HFCs": f"HFCS ({gwp_to_use})",
         "PFCs": f"PFCS ({gwp_to_use})",
         "SF6": f"SF6 ({gwp_to_use})",
-        "other halogenated gases" : f"other halogenated gases ({gwp_to_use})",
+        "other halogenated gases": f"other halogenated gases ({gwp_to_use})",
         "NMVOCs": "NMVOC",
     },
 }
@@ -184,3 +185,50 @@ meta_data = {
     "comment": "Read fom pdf by Daniel Busch",
     "institution": "UNFCCC",
 }
+
+country_processing_step1 = {
+    "aggregate_cats": {
+        "M.3.C.AG": {
+            "sources": [
+                "3.C.1",
+                "3.C.2",
+                "3.C.3",
+                "3.C.4",
+                "3.C.5",
+                "3.C.6",
+                "3.C.7",
+                "3.C.8",
+            ],
+            "name": "Aggregate sources and non-CO2 emissions sources on land "
+            "(Agriculture)",
+        },
+        "M.3.D.AG": {"sources": ["3.D.2"], "name": "Other (Agriculture)"},
+        "M.AG.ELV": {
+            "sources": ["M.3.C.AG", "M.3.D.AG"],
+            "name": "Agriculture excluding livestock",
+        },
+        "M.AG": {"sources": ["3.A", "M.AG.ELV"], "name": "Agriculture"},
+        "M.3.D.LU": {"sources": ["3.D.1"], "name": "Other (LULUCF)"},
+        "M.LULUCF": {"sources": ["3.B", "M.3.D.LU"], "name": "LULUCF"},
+        "M.0.EL": {
+            "sources": ["1", "2", "M.AG", "4", "5"],
+            "name": "National total emissions excluding LULUCF",
+        },
+    },
+    "basket_copy": {
+        "GWPs_to_add": ["AR4GWP100", "AR5GWP100", "AR6GWP100"],
+        "entities": ["HFCS", "PFCS"],
+        "source_GWP": gwp_to_use,
+    },
+}
+
+gas_baskets = {
+    "FGASES (SARGWP100)": ["HFCS (SARGWP100)", "PFCS (SARGWP100)", "SF6", "NF3"],
+    "FGASES (AR4GWP100)": ["HFCS (AR4GWP100)", "PFCS (AR4GWP100)", "SF6", "NF3"],
+    "FGASES (AR5GWP100)": ["HFCS (AR5GWP100)", "PFCS (AR5GWP100)", "SF6", "NF3"],
+    "FGASES (AR6GWP100)": ["HFCS (AR6GWP100)", "PFCS (AR6GWP100)", "SF6", "NF3"],
+    "KYOTOGHG (SARGWP100)": ["CO2", "CH4", "N2O", "FGASES (SARGWP100)"],
+    "KYOTOGHG (AR4GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR4GWP100)"],
+    "KYOTOGHG (AR5GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR5GWP100)"],
+    "KYOTOGHG (AR6GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR6GWP100)"],
+}

UNFCCC_GHG_data/UNFCCC_reader/Mongolia/read_MNG_BUR2_from_pdf.py

@@ -8,7 +8,7 @@ import camelot
 import primap2 as pm2
 import pandas as pd
 
-from UNFCCC_GHG_data.helper import downloaded_data_path, extracted_data_path, fix_rows
+from UNFCCC_GHG_data.helper import downloaded_data_path, extracted_data_path, fix_rows, process_data_for_country
 from config_MNG_BUR2 import (
     inv_conf,
     inv_conf_per_year,
@@ -18,6 +18,8 @@ from config_MNG_BUR2 import (
     coords_value_mapping,
     filter_remove,
     meta_data,
+    country_processing_step1,
+    gas_baskets
 )
 
 # ###
@@ -171,3 +173,56 @@ df_all_IF = pm2.pm2io.convert_long_dataframe_if(
 ### convert to primap2 format ###
 print("Converting to primap2 format.")
 data_pm2 = pm2.pm2io.from_interchange_format(df_all_IF)
+
+# ###
+# Save raw data to IF and native format.
+# ###
+
+data_if = data_pm2.pr.to_interchange_format()
+
+pm2.pm2io.write_interchange_format(
+    output_folder / (output_filename + coords_terminologies["category"] + "_raw"),
+    data_if,
+)
+
+encoding = {var: compression for var in data_pm2.data_vars}
+data_pm2.pr.to_netcdf(
+    output_folder / (output_filename + coords_terminologies["category"] + "_raw.nc"),
+    encoding=encoding,
+)
+
+# ###
+# Processing
+# ###
+
+data_proc_pm2 = process_data_for_country(
+    data_country=data_pm2,
+    entities_to_ignore=[],
+    gas_baskets=gas_baskets,
+    filter_dims=None,
+    cat_terminology_out=None,
+    category_conversion=None,
+    sectors_out=None,
+    processing_info_country=country_processing_step1,
+)
+
+# ###
+# save processed data to IF and native format
+# ###
+
+terminology_proc = coords_terminologies["category"]
+
+data_proc_if = data_proc_pm2.pr.to_interchange_format()
+
+if not output_folder.exists():
+    output_folder.mkdir()
+pm2.pm2io.write_interchange_format(
+    output_folder / (output_filename + terminology_proc), data_proc_if
+)
+
+encoding = {var: compression for var in data_proc_pm2.data_vars}
+data_proc_pm2.pr.to_netcdf(
+    output_folder / (output_filename + terminology_proc + ".nc"), encoding=encoding
+)
+
+print("Saved processed data.")