processing: category aggregation consistency checks and gas baskets

src/unfccc_ghg_data/unfccc_reader/Cabo_Verde/config_cpv_bur1.py

@@ -180,3 +180,58 @@ inv_conf_main = {
         },
     },
 }
+
+
+country_processing_step1 = {
+    # rounding error 0.038 for yr2019/entN2O/cat4: 0.011 + 0.015 != 0.027
+    "tolerance": 0.04,
+    "aggregate_cats": {
+        # Only 3.C.7 in table, but values are all zero or empty
+        # "M.3.C.AG": {  # "Aggregate sources and non-CO2 emissions sources on land (Agriculture)"
+        #     "sources": [
+        #         "3.C.3",
+        #         "3.C.4",
+        #         "3.C.5",
+        #         "3.C.7",
+        #     ]
+        # },
+        # 3.D.2 is all zeros
+        # "M.3.D.AG": {"sources": ["3.D.2"]},
+        # "M.3.D.AG" is empty, so I'm not sure we need it
+        # "M.AG.ELV": {
+        #     "sources": ["M.3.C.AG"],
+        # },
+        # M.AG = 3.A in this case
+        # "M.AG": {"sources": ["3.A", "M.AG.ELV"]},  # agriculture
+        # We don't have 3.D
+        # "M.LULUCF": {"sources": ["3.B"]},
+        # "M.0.EL": {
+        #     "sources": ["1", "2", "M.AG", "4"],
+        # },
+        # "1.B": {"sources": ["1.B.2"]},
+        "4.D": {"sources": ["4.D.1", "4.D.2"]},  # consistency check
+        "1": {"sources": ["1.A"]},  # consistency check
+        "1.A": {
+            "sources": ["1.A.1", "1.A.2", "1.A.3", "1.A.4", "1.A.5"]
+        },  # consistency check
+        "2": {
+            "sources": ["2.A", "2.B", "2.C", "2.D", "2.E", "2.F", "2.G", "2.H"]
+        },  # consistency check
+        "3": {"sources": ["M.AG", "M.LULUCF"]},  # consistency check
+        # "3.A": {"sources": ["3.A.1", "3.A.2"]}, # consistency check
+        "4": {"sources": ["4.A", "4.B", "4.C", "4.D", "4.E"]},  # consistency check
+    },
+    # We don't have HFCs and PFCs in the report, hence basket_copy is not relevant
+    # "basket_copy": {
+    #     "GWPs_to_add": ["SARGWP100", "AR5GWP100", "AR6GWP100"],
+    #     # "entities": ["HFCS", "PFCS"],
+    #     "source_GWP": gwp_to_use,
+    # },
+}
+
+gas_baskets = {
+    "KYOTOGHG (SARGWP100)": ["CO2", "CH4", "N2O"],
+    "KYOTOGHG (AR4GWP100)": ["CO2", "CH4", "N2O"],
+    "KYOTOGHG (AR5GWP100)": ["CO2", "CH4", "N2O"],
+    "KYOTOGHG (AR6GWP100)": ["CO2", "CH4", "N2O"],
+}

src/unfccc_ghg_data/unfccc_reader/Cabo_Verde/read_CPV_BUR1_from_pdf.py

@@ -7,13 +7,18 @@ import numpy as np
 import pandas as pd
 import primap2 as pm2
 
-from unfccc_ghg_data.helper import downloaded_data_path, extracted_data_path
+from unfccc_ghg_data.helper import (
+    downloaded_data_path,
+    extracted_data_path,
+    process_data_for_country,
+)
 from unfccc_ghg_data.unfccc_reader.Cabo_Verde.config_cpv_bur1 import (
     coords_cols,
     coords_defaults,
     coords_terminologies,
     coords_value_mapping,
     coords_value_mapping_main,
+    country_processing_step1,
     filter_remove,
     inv_conf,
     inv_conf_main,
@@ -253,3 +258,51 @@ if __name__ == "__main__":
         / (output_filename + coords_terminologies["category"] + "_raw.nc"),
         encoding=encoding,
     )
+
+    # # ###
+    # # Processing
+    # # ###
+
+    # create the gas baskets before aggregating the categories
+    # data_proc_pm2_gas_baskets = process_data_for_country(
+    #     data_country=data_pm2,
+    #     entities_to_ignore=[],
+    #     gas_baskets=gas_baskets,
+    #     filter_dims=None,
+    #     cat_terminology_out=None,
+    #     category_conversion=None,
+    #     sectors_out=None,
+    #     processing_info_country=None,
+    # )
+
+    data_proc_pm2 = process_data_for_country(
+        data_country=data_pm2,
+        entities_to_ignore=[],
+        gas_baskets=None,
+        filter_dims=None,
+        cat_terminology_out=None,
+        category_conversion=None,
+        sectors_out=None,
+        processing_info_country=country_processing_step1,
+    )
+
+    # # ###
+    # # save processed data to IF and native format
+    # # ###
+
+    terminology_proc = coords_terminologies["category"]
+
+    data_proc_if = data_proc_pm2.pr.to_interchange_format()
+
+    if not output_folder.exists():
+        output_folder.mkdir()
+    pm2.pm2io.write_interchange_format(
+        output_folder / (output_filename + terminology_proc), data_proc_if
+    )
+
+    encoding = {var: compression for var in data_proc_pm2.data_vars}
+    data_proc_pm2.pr.to_netcdf(
+        output_folder / (output_filename + terminology_proc + ".nc"), encoding=encoding
+    )
+
+    print("Saved processed data.")