Преглед изворни кода

[DATALAD] Recorded changes

Daniel Busch пре 2 дана
родитељ
комит
e54fcfca3f

+ 1 - 0
extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13.csv

@@ -0,0 +1 @@
+../../.git/annex/objects/9q/VQ/MD5E-s10232750--a1e11b106edb0841bb19d7e04a65bb69.csv/MD5E-s10232750--a1e11b106edb0841bb19d7e04a65bb69.csv

+ 1 - 0
extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13.nc

@@ -0,0 +1 @@
+../../.git/annex/objects/59/VF/MD5E-s3250602--1b5cae13af1c9fdb2e532c141dd36503.nc/MD5E-s3250602--1b5cae13af1c9fdb2e532c141dd36503.nc

+ 23 - 0
extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13.yaml

@@ -0,0 +1,23 @@
+attrs:
+  area: area (ISO3)
+  cat: category (IPCC2006_PRIMAP)
+  comment: Published by Food and Agriculture Organization of the United Nations (FAO),
+    converted to PRIMAP2 format by Daniel Busch
+  contact: daniel.busch@climate-resource.com
+  entity: N2O
+  institution: Food and Agriculture Organization of the United Nations
+  references: https://www.fao.org/faostat
+  rights: Creative Commons Attribution-4.0 International licence (CC BY 4.0)
+  scen: scenario (FAO)
+  title: Agrifood systems emissions
+data_file: FAOSTAT_Agrifood_system_emissions_v2023-12-13.csv
+dimensions:
+  '*':
+  - area (ISO3)
+  - category (IPCC2006_PRIMAP)
+  - entity
+  - scenario (FAO)
+  - source
+  - time
+  - unit
+time_format: '%Y'

+ 1 - 0
extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13_raw.csv

@@ -0,0 +1 @@
+../../.git/annex/objects/jQ/9f/MD5E-s32990120--9b50fb64b61dd771fd55940f9722967a.csv/MD5E-s32990120--9b50fb64b61dd771fd55940f9722967a.csv

+ 1 - 0
extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13_raw.nc

@@ -0,0 +1 @@
+../../.git/annex/objects/WJ/J0/MD5E-s13246774--9d03cc903c7ef543debd02c54536c45f.nc/MD5E-s13246774--9d03cc903c7ef543debd02c54536c45f.nc

+ 22 - 0
extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13_raw.yaml

@@ -0,0 +1,22 @@
+attrs:
+  area: area (ISO3)
+  cat: category (FAO)
+  comment: Published by Food and Agriculture Organization of the United Nations (FAO),
+    converted to PRIMAP2 format by Daniel Busch
+  contact: daniel.busch@climate-resource.com
+  institution: Food and Agriculture Organization of the United Nations
+  references: https://www.fao.org/faostat
+  rights: Creative Commons Attribution-4.0 International licence (CC BY 4.0)
+  scen: scenario (FAO)
+  title: Agrifood systems emissions
+data_file: FAOSTAT_Agrifood_system_emissions_v2023-12-13_raw.csv
+dimensions:
+  '*':
+  - area (ISO3)
+  - category (FAO)
+  - entity
+  - scenario (FAO)
+  - source
+  - time
+  - unit
+time_format: '%Y'

+ 1 - 1
extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14.csv

@@ -1 +1 @@
-../../.git/annex/objects/53/g3/MD5E-s10460115--8e343b2523d4e0e152fd9f4fefaf674e.csv/MD5E-s10460115--8e343b2523d4e0e152fd9f4fefaf674e.csv
+../../.git/annex/objects/3G/Xw/MD5E-s10455053--2d3b5df6f172a6393c9334b6783ed0cc.csv/MD5E-s10455053--2d3b5df6f172a6393c9334b6783ed0cc.csv

+ 1 - 1
extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14.nc

@@ -1 +1 @@
-../../.git/annex/objects/K2/Vx/MD5E-s3326796--dce71b6983c5a9c4463acf571e8c6fbc.nc/MD5E-s3326796--dce71b6983c5a9c4463acf571e8c6fbc.nc
+../../.git/annex/objects/xx/23/MD5E-s3329305--a338fd39d4c8e44364f7c5e0267d71f5.nc/MD5E-s3329305--a338fd39d4c8e44364f7c5e0267d71f5.nc

+ 1 - 1
extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.csv

@@ -1 +1 @@
-../../.git/annex/objects/jP/q8/MD5E-s33059801--f3de61280a18010028e160b9d685aaeb.csv/MD5E-s33059801--f3de61280a18010028e160b9d685aaeb.csv
+../../.git/annex/objects/xv/3j/MD5E-s33059797--a79a80ee8a3b836f7c376a4d23dfbef4.csv/MD5E-s33059797--a79a80ee8a3b836f7c376a4d23dfbef4.csv

+ 1 - 1
extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.nc

@@ -1 +1 @@
-../../.git/annex/objects/pg/MW/MD5E-s14608047--7356c5b93616ebfd7059f5c44c795ca1.nc/MD5E-s14608047--7356c5b93616ebfd7059f5c44c795ca1.nc
+../../.git/annex/objects/37/9w/MD5E-s14515527--56222b762f2646fe6e0e6b4f1e9088fc.nc/MD5E-s14515527--56222b762f2646fe6e0e6b4f1e9088fc.nc

+ 2 - 2
extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.yaml

@@ -1,6 +1,6 @@
 attrs:
   area: area (ISO3)
-  cat: category (FAOSTAT)
+  cat: category (FAO)
   comment: Published by Food and Agriculture Organization of the United Nations (FAO),
     converted to PRIMAP2 format by Daniel Busch
   contact: daniel.busch@climate-resource.com
@@ -13,7 +13,7 @@ data_file: FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.csv
 dimensions:
   '*':
   - area (ISO3)
-  - category (FAOSTAT)
+  - category (FAO)
   - entity
   - scenario (FAO)
   - source

+ 19 - 18
src/faostat_data_primap/helper/category_aggregation.py

@@ -9,7 +9,7 @@ Definitions for category aggregation.
 # At the moment, we are only checking categories that will later be used by primap-hist.
 # If we want to use other categories we should expand this consistency check.
 agg_info_fao = {
-    "category (FAOSTAT)": {
+    "category (FAO)": {
         # 1.A.1 wheat
         # rounding errors
         "1.A.1.a": {
@@ -21,7 +21,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.1": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.1.a",
                 "1.A.1.b",
@@ -39,7 +39,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.2": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.2.a",
                 "1.A.2.b",
@@ -57,7 +57,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.3": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.3.a",
             ],
@@ -73,7 +73,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.4": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.4.a",
             ],
@@ -105,7 +105,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.6": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.6.a",
                 "1.A.6.b",
@@ -122,7 +122,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.7": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.7.a",
                 "1.A.7.b",
@@ -139,7 +139,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.8": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.8.a",
             ],
@@ -155,7 +155,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.9": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.9.a",
             ],
@@ -171,7 +171,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.10": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.10.a",
             ],
@@ -187,7 +187,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.11": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.11.a",
             ],
@@ -203,7 +203,7 @@ agg_info_fao = {
             "sel": {"variable": ["N2O"]},
         },
         "1.A.12": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.A.12.a",
             ],
@@ -228,7 +228,7 @@ agg_info_fao = {
             ],
         },
         "1.B.2": {
-            "tolerance": 0.01,
+            "tolerance": 1,
             "sources": [
                 "1.B.2.a",
                 "1.B.2.b",
@@ -243,7 +243,7 @@ agg_info_fao = {
             ],
             "sel": {"variable": ["N2O"]},
         },
-        # Category 1 is not available on FAOSTAT, so that's not a check
+        # Category 1 is not available on FAOS, so that's not a check
         "1": {
             "tolerance": 0.01,
             "sources": [
@@ -389,13 +389,14 @@ agg_info_ipcc2006_primap = {
         },
         "M.3.C.AG": {
             "sources": [
-                "3.C.1.b",  # Biomass Burning In Croplands
-                "3.C.1.c",  # Biomass Burning in Grasslands
+                "3.C.1.b",  # Biomass Burning In Croplands - looks good (CH4, N2O)
+                "3.C.1.c",  # Biomass Burning in Grasslands - looks good (CH4)
                 "3.C.4",  # Direct N2O Emissions from Managed Soils
                 "M.3.C.4.SF",  # synthetic fertilisers
                 "3.C.5",  # Indirect N2O Emissions from Managed Soils
                 "M.3.C.5.SF",  # synthetic fertilisers
                 "3.C.6",  # Indirect N2O Emissions from Manure Management
+                "3.C.7",  # rice cultivation
             ],
         },
         "M.AG.ELV": {
@@ -406,9 +407,9 @@ agg_info_ipcc2006_primap = {
                 "3.C.1",
                 "3.C.2",
                 "3.C.3",
-                "3.C.4",
+                "3.C.4",  # excluding synthetic fertilisers
                 "M.3.C.4.SF",  # synthetic fertilisers
-                "3.C.5",
+                "3.C.5",  # excluding synthetic fertilisers
                 "M.3.C.5.SF",  # synthetic fertilisers
                 "3.C.6",
                 "3.C.7",

+ 446 - 6
src/faostat_data_primap/helper/definitions.py

@@ -83,6 +83,52 @@ areas_to_remove_base = [
 # TODO would be a nice to have a type hint here
 read_config_all: Any = {
     "farm_gate_agriculture_energy": {
+        "2023-12-13": {
+            "filename": "Emissions_Agriculture_Energy_E_All_Data_NOFLAG.csv",
+            "areas_to_remove": [
+                *areas_to_remove_base,
+            ],
+            "elements_to_remove": ["Energy use in agriculture"],
+            "entity_mapping": {
+                "Emissions (CO2)": "CO2",
+                "Emissions (CH4)": "CH4",
+                "Emissions (N2O)": "N2O",
+            },
+            "category_mapping_item_element": {
+                "Electricity - Emissions (CO2)": "2.B",
+                "Electricity - Emissions (CH4)": "2.B",
+                "Electricity - Emissions (N2O)": "2.B",
+                "Total Energy - Emissions (CO2)": "2",
+                "Total Energy - Emissions (CH4)": "2",
+                "Total Energy - Emissions (N2O)": "2",
+                "Petroleum products - Emissions (CO2)": "2.E",
+                "Petroleum products - Emissions (CH4)": "2.E",
+                "Petroleum products - Emissions (N2O)": "2.E",
+                "Natural gas - Emissions (CO2)": "2.A",
+                "Natural gas - Emissions (CH4)": "2.A",
+                "Natural gas - Emissions (N2O)": "2.A",
+                "Coal - Emissions (CO2)": "2.C",
+                "Coal - Emissions (CH4)": "2.C",
+                "Coal - Emissions (N2O)": "2.C",
+                "Heat - Emissions (CO2)": "2.D",
+                "Heat - Emissions (CH4)": "2.D",
+                "Heat - Emissions (N2O)": "2.D",
+            },
+            "items-elements_to_remove": [
+                "Total Energy (excl.eletricity & heat) - Emissions (CO2)",
+                "Total Energy (excl.eletricity & heat) - Emissions (CH4)",
+                "Total Energy (excl.eletricity & heat) - Emissions (N2O)",
+            ],
+            "columns_to_drop": [
+                "Element",
+                "Element Code",
+                "Item",
+                "Item Code",
+                "Area Code (M49)",
+                "Area",
+                "Area Code",
+            ],
+        },
         "2024-11-14": {
             "filename": "Emissions_Agriculture_Energy_E_All_Data_NOFLAG.csv",
             "areas_to_remove": [
@@ -128,9 +174,146 @@ read_config_all: Any = {
                 "Area",
                 "Area Code",
             ],
-        }
+        },
     },
     "farm_gate_emissions_crops": {
+        "2023-11-09": {
+            "filename": "Emissions_crops_E_All_Data_NOFLAG.csv",
+            "areas_to_remove": [
+                *areas_to_remove_base,
+                "European Union (27)",
+            ],
+            "elements_to_remove": [
+                "Crop residues (N content)",
+                "Burning crop residues (Biomass burned, dry matter)",
+                "Area harvested",
+                "Nitrogen fertilizer content applied that leaches",
+                "Nitrogen fertilizer content applied that volatilises",
+                "Synthetic fertilizers (Agricultural use)",
+            ],
+            "entity_mapping": {
+                "Crop residues (Emissions N2O)": "N2O",
+                "Crop residues (Direct emissions N2O)": "N2O",
+                "Crop residues (Indirect emissions N2O)": "N2O",
+                "Burning crop residues (Emissions N2O)": "N2O",
+                "Burning crop residues (Emissions CH4)": "CH4",
+                "Rice cultivation (Emissions CH4)": "CH4",
+                "Crops total (Emissions N2O)": "N2O",
+                "Crops total (Emissions CH4)": "CH4",
+                "Synthetic fertilizers (Emissions N2O)": "N2O",
+                "Synthetic fertilizers (Direct emissions N2O)": "N2O",
+                "Synthetic fertilizers (Indirect emissions N2O)": "N2O",
+                "Indirect emissions (N2O that leaches) (Synthetic fertilizers)": "N2O",
+                (
+                    "Indirect emissions (N2O that volatilises) "
+                    "(Synthetic fertilizers)"
+                ): "N2O",
+            },
+            "category_mapping_item_element": {
+                "Barley - Crop residues (Emissions N2O)": "1.A.5.a",
+                "Barley - Crop residues (Direct emissions N2O)": "1.A.5.a.i",
+                "Barley - Crop residues (Indirect emissions N2O)": "1.A.5.a.ii",
+                "Maize (corn) - Crop residues (Emissions N2O)": "1.A.6.a",
+                "Maize (corn) - Crop residues (Direct emissions N2O)": "1.A.6.a.i",
+                "Maize (corn) - Crop residues (Indirect emissions N2O)": "1.A.6.a.ii",
+                "Maize (corn) - Burning crop residues (Emissions N2O)": "1.A.6.b",
+                "Maize (corn) - Burning crop residues (Emissions CH4)": "1.A.6.b",
+                "Sugar cane - Crop residues (Emissions N2O)": "1.A.7.a",
+                "Sugar cane - Burning crop residues (Emissions N2O)": "1.A.7.b",
+                "Sugar cane - Burning crop residues (Emissions CH4)": "1.A.7.b",
+                "Millet - Crop residues (Emissions N2O)": "1.A.4.a",
+                "Millet - Crop residues (Direct emissions N2O)": "1.A.4.a.i",
+                "Millet - Crop residues (Indirect emissions N2O)": "1.A.4.a.ii",
+                "Potatoes - Crop residues (Emissions N2O)": "1.A.3.a",
+                "Potatoes - Crop residues (Direct emissions N2O)": "1.A.3.a.i",
+                "Potatoes - Crop residues (Indirect emissions N2O)": "1.A.3.a.ii",
+                "Rice - Crop residues (Emissions N2O)": "1.A.2.a",
+                "Rice - Crop residues (Direct emissions N2O)": "1.A.2.a.i",
+                "Rice - Crop residues (Indirect emissions N2O)": "1.A.2.a.ii",
+                "Rice - Burning crop residues (Emissions N2O)": "1.A.2.b",
+                "Rice - Burning crop residues (Emissions CH4)": "1.A.2.b",
+                "Rice - Rice cultivation (Emissions CH4)": "1.A.2.c",
+                "Wheat - Crop residues (Emissions N2O)": "1.A.1.a",
+                "Wheat - Crop residues (Direct emissions N2O)": "1.A.1.a.i",
+                "Wheat - Crop residues (Indirect emissions N2O)": "1.A.1.a.ii",
+                "Wheat - Burning crop residues (Emissions N2O)": "1.A.1.b",
+                "Wheat - Burning crop residues (Emissions CH4)": "1.A.1.b",
+                "All Crops - Crops total (Emissions N2O)": "1.A",
+                "All Crops - Crops total (Emissions CH4)": "1.A",
+                (
+                    "Nutrient nitrogen N (total) - Synthetic "
+                    "fertilizers (Emissions N2O)"
+                ): "1.B",
+                (
+                    "Nutrient nitrogen N (total) - "
+                    "Synthetic fertilizers (Direct emissions N2O)"
+                ): "1.B.1",
+                (
+                    "Nutrient nitrogen N (total) - "
+                    "Synthetic fertilizers (Indirect emissions N2O)"
+                ): "1.B.2",
+                (
+                    "Nutrient nitrogen N (total) - "
+                    "Indirect emissions (N2O that leaches) "
+                    "(Synthetic fertilizers)"
+                ): "1.B.2.b",
+                (
+                    "Nutrient nitrogen N (total) - "
+                    "Indirect emissions (N2O that volatilises) "
+                    "(Synthetic fertilizers)"
+                ): "1.B.2.a",
+                "Beans, dry - Crop residues (Emissions N2O)": "1.A.8.a",
+                "Beans, dry - Crop residues (Direct emissions N2O)": "1.A.8.a.i",
+                "Beans, dry - Crop residues (Indirect emissions N2O)": "1.A.8.a.ii",
+                "Oats - Crop residues (Emissions N2O)": "1.A.9.a",
+                "Oats - Crop residues (Direct emissions N2O)": "1.A.9.a.i",
+                "Oats - Crop residues (Indirect emissions N2O)": "1.A.9.a.ii",
+                "Rye - Crop residues (Emissions N2O)": "1.A.10.a",
+                "Rye - Crop residues (Direct emissions N2O)": "1.A.10.a.i",
+                "Rye - Crop residues (Indirect emissions N2O)": "1.A.10.a.ii",
+                "Sorghum - Crop residues (Emissions N2O)": "1.A.11.a",
+                "Sorghum - Crop residues (Direct emissions N2O)": "1.A.11.a.i",
+                "Sorghum - Crop residues (Indirect emissions N2O)": "1.A.11.a.ii",
+                "Soya beans - Crop residues (Emissions N2O)": "1.A.12.a",
+                "Soya beans - Crop residues (Direct emissions N2O)": "1.A.12.a.i",
+                "Soya beans - Crop residues (Indirect emissions N2O)": "1.A.12.a.ii",
+                "Maize (corn) - Crops total (Emissions N2O)": "1.A.6",
+                "Maize (corn) - Crops total (Emissions CH4)": "1.A.6",
+                "Sugar cane - Crops total (Emissions N2O)": "1.A.7",
+                "Sugar cane - Crops total (Emissions CH4)": "1.A.7",
+                "Rice - Crops total (Emissions N2O)": "1.A.2",
+                "Rice - Crops total (Emissions CH4)": "1.A.2",
+                "Wheat - Crops total (Emissions N2O)": "1.A.1",
+                "Wheat - Crops total (Emissions CH4)": "1.A.1",
+                # these extra categories are NOT in the 2024 release
+                "Barley - Crops total (Emissions N2O)": "1.A.5",
+                "Millet - Crops total (Emissions N2O)": "1.A.4",
+                "Potatoes - Crops total (Emissions N2O)": "1.A.3",
+                "Beans, dry - Crops total (Emissions N2O)": "1.A.8",
+                "Oats - Crops total (Emissions N2O)": "1.A.9",
+                "Rye - Crops total (Emissions N2O)": "1.A.10",
+                "Sorghum - Crops total (Emissions N2O)": "1.A.11",
+                "Soya beans - Crops total (Emissions N2O)": "1.A.12",
+            },
+            "items-elements_to_remove": [
+                "All Crops - Crop residues (Emissions N2O)",
+                "All Crops - Crop residues (Direct emissions N2O)",
+                "All Crops - Crop residues (Indirect emissions N2O)",
+                "All Crops - Burning crop residues (Emissions N2O)",
+                "All Crops - Burning crop residues (Emissions CH4)",
+            ],
+            "columns_to_drop": [
+                "Element",
+                "Element Code",
+                "Item",
+                "Item Code",
+                "Area Code (M49)",
+                "Area",
+                "Area Code",
+                "Item Code (CPC)",
+                "Source Code",
+            ],
+        },
         "2024-11-14": {
             "filename": "Emissions_crops_E_All_Data_NOFLAG.csv",
             "areas_to_remove": [
@@ -253,9 +436,142 @@ read_config_all: Any = {
                 "Item Code (CPC)",
                 "Source Code",
             ],
-        }
+        },
     },
     "farm_gate_livestock": {
+        "2023-11-09": {
+            "filename": "Emissions_livestock_E_All_Data_NOFLAG.csv",
+            "areas_to_remove": [
+                *areas_to_remove_base,
+                "Belgium-Luxembourg",
+                "Serbia and Montenegro",
+                "European Union (27)",
+            ],
+            "items_to_remove": [
+                # we don't need aggregates
+                "Camels and Llamas",
+                "Cattle",  # dairy and non-dairy
+                # mistake by FAO, should be "Mules, hinnies, and asses"
+                "Mules and Asses",
+                "Sheep and Goats",
+                "Swine",  # breeding and market
+            ],
+            "elements_to_remove": [
+                "Stocks",  # number of animals
+                "Manure management (manure treated, N content)",
+                "Manure left on pasture (N content)",
+                "Manure left on pasture that leaches (N content)",
+                "Manure left on pasture that volatilises (N content)",
+                "Manure applied to soils (N content)",
+                "Manure applied to soils that leaches (N content)",
+                "Manure applied to soils that volatilises (N content)",
+                # TODO ?
+                # sum of direct and direct manure management emissions
+                # would add another level in the category tree, but
+                # is not needed (see miro)
+                # "Manure management (Emissions N2O)",
+            ],
+            "entity_mapping": {
+                "Livestock total (Emissions N2O)": "N2O",
+                "Livestock total (Emissions CH4)": "CH4",
+                "Enteric fermentation (Emissions CH4)": "CH4",
+                "Manure management (Emissions CH4)": "CH4",
+                "Manure management (Emissions N2O)": "N2O",
+                "Manure management (Direct emissions N2O)": "N2O",
+                "Manure management (Indirect emissions N2O)": "N2O",
+                "Manure left on pasture (Emissions N2O)": "N2O",
+                "Manure left on pasture (Direct emissions N2O)": "N2O",
+                "Indirect emissions (N2O that leaches) (Manure on pasture)": "N2O",
+                "Indirect emissions (N2O that volatilises) (Manure on pasture)": "N2O",
+                "Manure left on pasture (Indirect emissions N2O)": "N2O",
+                "Emissions (N2O) (Manure applied)": "N2O",
+                "Manure applied to soils (Direct emissions N2O)": "N2O",
+                "Indirect emissions (N2O that leaches) (Manure applied)": "N2O",
+                "Indirect emissions (N2O that volatilises) (Manure applied)": "N2O",
+                "Manure applied to soils (Indirect emissions N2O)": "N2O",
+            },
+            "category_mapping_item": {
+                "All Animals": "3",
+                "Asses": "3.A",
+                "Camels": "3.B",
+                "Cattle, dairy": "3.C",
+                "Cattle, non-dairy": "3.D",
+                "Chickens, broilers": "3.E",
+                "Chickens, layers": "3.F",
+                "Goats": "3.G",
+                "Horses": "3.H",
+                "Mules and hinnies": "3.I",
+                "Sheep": "3.J",
+                "Llamas": "3.K",
+                "Chickens": "3.L",
+                "Poultry Birds": "3.M",
+                "Buffalo": "3.N",
+                "Ducks": "3.O",
+                "Swine, breeding": "3.P",
+                "Swine, market": "3.Q",
+                "Turkeys": "3.R",
+            },
+            "category_mapping_element": {
+                "Livestock total (Emissions N2O)": "",
+                "Livestock total (Emissions CH4)": "",
+                "Enteric fermentation (Emissions CH4)": ".4",
+                "Manure management (Emissions CH4)": ".1.a",
+                # TODO we need to aggregate 3.X.1 for CH4
+                "Manure management (Emissions N2O)": ".1",
+                "Manure management (Direct emissions N2O)": ".1.b",
+                "Manure management (Indirect emissions N2O)": ".1.c",
+                "Manure left on pasture (Emissions N2O)": ".2",
+                "Manure left on pasture (Direct emissions N2O)": ".2.a",
+                "Indirect emissions (N2O that leaches) (Manure on pasture)": ".2.b.i",
+                (
+                    "Indirect emissions (N2O that volatilises) " "(Manure on pasture)"
+                ): ".2.b.ii",
+                "Manure left on pasture (Indirect emissions N2O)": ".2.b",
+                "Emissions (N2O) (Manure applied)": ".3",
+                "Manure applied to soils (Direct emissions N2O)": ".3.a",
+                ("Indirect emissions (N2O that leaches) " "(Manure applied)"): ".3.b.i",
+                "Indirect emissions (N2O that volatilises) (Manure applied)": ".3.b.ii",
+                "Manure applied to soils (Indirect emissions N2O)": ".3.b",
+            },
+            "items-elements_to_remove": [
+                # we only keep All animals total CH4 and total N2O
+                "All Animals - Enteric fermentation (Emissions CH4)",
+                "All Animals - Manure management (Emissions CH4)",
+                "All Animals - Manure management (Direct emissions N2O)",
+                "All Animals - Manure management (Indirect emissions N2O)",
+                "All Animals - Manure left on pasture (Emissions N2O)",
+                "All Animals - Manure left on pasture (Direct emissions N2O)",
+                (
+                    "All Animals - Indirect emissions (N2O that leaches) "
+                    "(Manure on pasture)"
+                ),
+                (
+                    "All Animals - Indirect emissions (N2O that volatilises) "
+                    "(Manure on pasture)"
+                ),
+                "All Animals - Manure left on pasture (Indirect emissions N2O)",
+                "All Animals - Emissions (N2O) (Manure applied)",
+                "All Animals - Manure applied to soils (Direct emissions N2O)",
+                "All Animals - Indirect emissions (N2O that leaches) (Manure applied)",
+                (
+                    "All Animals - Indirect emissions (N2O that volatilises) "
+                    "(Manure applied)"
+                ),
+                "All Animals - Manure applied to soils (Indirect emissions N2O)",
+                "All Animals - Manure management (Emissions N2O)",
+            ],
+            "columns_to_drop": [
+                "Element",
+                "Element Code",
+                "Item",
+                "Item Code",
+                "Area Code (M49)",
+                "Area",
+                "Area Code",
+                "Item Code (CPC)",
+                "Source Code",
+            ],
+        },
         "2024-11-14": {
             "filename": "Emissions_livestock_E_All_Data_NOFLAG.csv",
             "areas_to_remove": [
@@ -388,13 +704,15 @@ read_config_all: Any = {
                 "Item Code (CPC)",
                 "Source Code",
             ],
-        }
+        },
     },
     "land_use_drained_organic_soils": {
         "2023-11-09": {
             "filename": "Emissions_Drained_Organic_Soils_E_All_Data_NOFLAG.csv",
             "areas_to_remove": [
                 *areas_to_remove_base,
+                # "Belgium-Luxembourg",
+                # "Serbia and Montenegro",
                 "European Union (27)",
             ],
             "elements_to_remove": [
@@ -415,6 +733,14 @@ read_config_all: Any = {
                 "Area Code",
                 "Source Code",
             ],
+            "category_mapping_item_element": {
+                "Cropland organic soils - Emissions (N2O)": "5.B",
+                "Cropland organic soils - Emissions (CO2)": "5.B",
+                "Grassland organic soils - Emissions (N2O)": "5.A",
+                "Grassland organic soils - Emissions (CO2)": "5.A",
+                "Drained organic soils - Emissions (N2O)": "5",
+                "Drained organic soils - Emissions (CO2)": "5",
+            },
         },
         "2024-11-14": {
             "filename": "Emissions_Drained_Organic_Soils_E_All_Data_NOFLAG.csv",
@@ -459,12 +785,43 @@ read_config_all: Any = {
                 *areas_to_remove_base,
                 "European Union (27)",
             ],
-            "elements_to_remove": ["Biomass burned (dry matter)", "Burned Area"],
+            "elements_to_remove": [
+                "Burning crop residues (Biomass burned, dry matter)",
+                "Burned Area",
+                # Not in 2024 data set
+                "Biomass burned (dry matter)",
+            ],
             "entity_mapping": {
                 "Emissions (CH4)": "CH4",
                 "Emissions (N2O)": "N2O",
                 "Emissions (CO2)": "CO2",
             },
+            "category_mapping_item_element": {
+                "Forest fires - Emissions (CH4)": "6.A",
+                "Forest fires - Emissions (N2O)": "6.A",
+                "Forest fires - Emissions (CO2)": "6.A",
+                "Humid tropical forest - Emissions (CH4)": "6.A.1",
+                "Humid tropical forest - Emissions (N2O)": "6.A.1",
+                "Other forest - Emissions (CH4)": "6.A.2",
+                "Other forest - Emissions (N2O)": "6.A.2",
+                "Other forest - Emissions (CO2)": "6.A.2",
+                "Savanna fires - Emissions (CH4)": "6.B",
+                "Savanna fires - Emissions (N2O)": "6.B",
+                "Closed shrubland - Emissions (CH4)": "6.B.1",
+                "Closed shrubland - Emissions (N2O)": "6.B.1",
+                "Grassland - Emissions (CH4)": "6.B.2",
+                "Grassland - Emissions (N2O)": "6.B.2",
+                "Open shrubland - Emissions (CH4)": "6.B.3",
+                "Open shrubland - Emissions (N2O)": "6.B.3",
+                "Savanna - Emissions (CH4)": "6.B.4",
+                "Savanna - Emissions (N2O)": "6.B.4",
+                "Woody savanna - Emissions (CH4)": "6.B.5",
+                "Woody savanna - Emissions (N2O)": "6.B.5",
+                "Fires in organic soils - Emissions (CH4)": "6.C",
+                "Fires in organic soils - Emissions (CO2)": "6.C",
+                "Humid tropical forest - Emissions (CO2)": "6.A.1",
+                "Savanna fires - Emissions (CO2)": "6.B",
+            },
             "columns_to_drop": [
                 "Element",
                 "Element Code",
@@ -530,6 +887,32 @@ read_config_all: Any = {
         },
     },
     "land_use_forests": {
+        "2023-11-09": {
+            "filename": "Emissions_Land_Use_Forests_E_All_Data_NOFLAG.csv",
+            "areas_to_remove": [
+                *areas_to_remove_base,
+                "European Union (27)",
+            ],
+            "elements_to_remove": [
+                "Area",
+            ],
+            "entity_mapping": {"Net emissions/removals (CO2) (Forest land)": "CO2"},
+            "category_mapping_item_element": {
+                "Forestland - Net emissions/removals (CO2) (Forest land)": "4.A",
+                "Net Forest conversion - Net emissions/removals (CO2) (Forest land)": "4.B",
+                "Carbon stock change in forests - Net emissions/removals (CO2) (Forest land)": "4",
+            },
+            "columns_to_drop": [
+                "Element",
+                "Element Code",
+                "Item",
+                "Item Code",
+                "Area Code (M49)",
+                "Area",
+                "Area Code",
+                "Source Code",
+            ],
+        },
         "2024-11-14": {
             "filename": "Emissions_Land_Use_Forests_E_All_Data_NOFLAG.csv",
             "areas_to_remove": [
@@ -555,7 +938,7 @@ read_config_all: Any = {
                 "Area Code",
                 "Source Code",
             ],
-        }
+        },
     },
     "pre_post_agricultural_production": {
         "2023-11-09": {
@@ -578,6 +961,63 @@ read_config_all: Any = {
                 "Emissions (N2O)": "N2O",
                 "Emissions (CO2eq) from F-gases (AR5)": "FGASES (AR5GWP100)",
             },
+            "category_mapping_item_element": {
+                "Fertilizers Manufacturing - Emissions (CO2)": "7.A",
+                "Fertilizers Manufacturing - Emissions (CO2eq) (AR5)": "7.A",
+                "Food Transport - Emissions (CO2)": "7.B",
+                "Food Transport - Emissions (CH4)": "7.B",
+                "Food Transport - Emissions (N2O)": "7.B",
+                "Food Transport - Emissions (CO2eq) (AR5)": "7.B",
+                "Food Retail - Emissions (CO2)": "7.C",
+                "Food Retail - Emissions (CH4)": "7.C",
+                "Food Retail - Emissions (N2O)": "7.C",
+                "Food Retail - Emissions (CO2eq) from F-gases (AR5)": "7.C",
+                "Food Retail - Emissions (CO2eq) (AR5)": "7.C",
+                "Food Household Consumption - Emissions (CO2)": "7.D",
+                "Food Household Consumption - Emissions (CH4)": "7.D",
+                "Food Household Consumption - Emissions (N2O)": "7.D",
+                "Food Household Consumption - Emissions (CO2eq) (AR5)": "7.D",
+                "Solid Food Waste - Emissions (CH4)": "7.E",
+                "Solid Food Waste - Emissions (CO2eq) (AR5)": "7.E",
+                "Domestic Wastewater - Emissions (CH4)": "7.F",
+                "Domestic Wastewater - Emissions (N2O)": "7.F",
+                "Domestic Wastewater - Emissions (CO2eq) (AR5)": "7.F",
+                "Industrial Wastewater - Emissions (CH4)": "7.G",
+                "Industrial Wastewater - Emissions (N2O)": "7.G",
+                "Industrial Wastewater - Emissions (CO2eq) (AR5)": "7.G",
+                "Incineration - Emissions (CO2)": "7.H",
+                "Incineration - Emissions (CO2eq) (AR5)": "7.H",
+                "Pre- and Post- Production - Emissions (CO2)": "7.I",
+                "Pre- and Post- Production - Emissions (CH4)": "7.I",
+                "Pre- and Post- Production - Emissions (N2O)": "7.I",
+                "Pre- and Post- Production - Emissions (CO2eq) from F-gases (AR5)": "7.I",
+                "Pre- and Post- Production - Emissions (CO2eq) (AR5)": "7.I",
+                "Energy Use (Pre- and Post-Production) - Emissions (CO2)": "7.J",
+                "Energy Use (Pre- and Post-Production) - Emissions (CH4)": "7.J",
+                "Energy Use (Pre- and Post-Production) - Emissions (N2O)": "7.J",
+                "Energy Use (Pre- and Post-Production) - Emissions (CO2eq) (AR5)": "7.J",
+                "Agrifood Systems Waste Disposal - Emissions (CO2)": "7.K",
+                "Agrifood Systems Waste Disposal - Emissions (CH4)": "7.K",
+                "Agrifood Systems Waste Disposal - Emissions (N2O)": "7.K",
+                "Agrifood Systems Waste Disposal - Emissions (CO2eq) (AR5)": "7.K",
+                "Cold Chain F-Gas - Emissions (CO2eq) from F-gases (AR5)": "7.L",
+                "Pesticides Manufacturing - Emissions (CO2)": "7.M",
+                "Pesticides Manufacturing - Emissions (CH4)": "7.M",
+                "Pesticides Manufacturing - Emissions (N2O)": "7.M",
+                "Pesticides Manufacturing - Emissions (CO2eq) (AR5)": "7.M",
+                "Food Processing - Emissions (CO2)": "7.N",
+                "Food Processing - Emissions (CH4)": "7.N",
+                "Food Processing - Emissions (N2O)": "7.N",
+                "Food Processing - Emissions (CO2eq) from F-gases (AR5)": "7.N",
+                "Food Processing - Emissions (CO2eq) (AR5)": "7.N",
+                "Food Transport - Emissions (CO2eq) from F-gases (AR5)": "7.B",
+                "Food Packaging - Emissions (CO2)": "7.O",
+                "Food Packaging - Emissions (CH4)": "7.O",
+                "Food Packaging - Emissions (N2O)": "7.O",
+                "Food Packaging - Emissions (CO2eq) (AR5)": "7.O",
+                "Food Household Consumption - Emissions (CO2eq) from F-gases (AR5)": "7.D",
+                "Fertilizers Manufacturing - Emissions (N2O)": "7.A",
+            },
             "columns_to_drop": [
                 "Element",
                 "Element Code",
@@ -690,7 +1130,7 @@ config_to_if = {
         "source": "Source",
         "category": "category",
     },
-    "coords_terminologies": {"area": "ISO3", "category": "FAOSTAT", "scenario": "FAO"},
+    "coords_terminologies": {"area": "ISO3", "category": "FAO", "scenario": "FAO"},
     "coords_value_mapping": {},
     "filter_keep": {},
     "filter_remove": {},

+ 3 - 3
src/faostat_data_primap/read.py

@@ -3,6 +3,7 @@
 import os
 import pathlib
 
+import climate_categories as cc
 import pandas as pd
 import primap2 as pm2  # type: ignore
 
@@ -13,7 +14,6 @@ from faostat_data_primap.helper.definitions import (
 )
 
 # todo replace with FAO climate categories once it's done
-from faostat_data_primap.helper.FAO_categorisation_temp import spec
 from faostat_data_primap.helper.paths import (
     downloaded_data_path,
     extracted_data_path,
@@ -177,14 +177,14 @@ def read_data(  # noqa: PLR0915 PLR0912
         )
 
         # check if all Item-Element combinations are now converted to category codes
-        fao_categories = list(spec["categories"].keys())
+        fao_categories = list(cc.FAO.df.index)
         unknown_categories = [
             i for i in df_domain.category.unique() if i not in fao_categories
         ]
         if unknown_categories:
             msg = (
                 f"Not all categories are part of FAO categorisation. "
-                f"Check mapping for {unknown_categories}"
+                f"Check mapping for {unknown_categories} in domain {domain}"
             )
             raise ValueError(msg)
 

+ 35 - 12
tests/unit/test_conversion.py

@@ -15,19 +15,21 @@ from src.faostat_data_primap.read import read_data
 
 def test_conversion_from_FAO_to_IPCC2006_PRIMAP():
     # make categorisation A from yaml
-    categorisation_a = cc.from_python("FAO.py")
+    categorisation_a = cc.FAO
     # make categorisation B from yaml
     categorisation_b = cc.IPCC2006_PRIMAP
 
     # category FAOSTAT not yet part of climate categories, so we need to add it manually
     cats = {
-        "FAOSTAT": categorisation_a,
+        "FAO": categorisation_a,
         "IPCC2006_PRIMAP": categorisation_b,
     }
-
+    # release_name = "v2024-11-14"
+    release_name = "v2023-12-13"
     ds_fao = (
         extracted_data_path
-        / "v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.nc"
+        # / "v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.nc"
+        / f"{release_name}/FAOSTAT_Agrifood_system_emissions_{release_name}_raw.nc"
     )
     ds = pm2.open_dataset(ds_fao)
 
@@ -51,7 +53,7 @@ def test_conversion_from_FAO_to_IPCC2006_PRIMAP():
     da_dict = {}
     for var in gases:
         da_dict[var] = ds[var].pr.convert(
-            dim="category (FAOSTAT)",
+            dim="category (FAO)",
             conversion=conv[var],
         )
     result = xr.Dataset(da_dict)
@@ -70,7 +72,6 @@ def test_conversion_from_FAO_to_IPCC2006_PRIMAP():
     result_proc_if = result_proc.pr.to_interchange_format()
 
     # save processed data
-    release_name = "v2024-11-14"
     output_filename = f"FAOSTAT_Agrifood_system_emissions_{release_name}"
     output_folder = extracted_data_path / release_name
 
@@ -109,6 +110,31 @@ def test_read(tmp_path):
     )
 
 
+def test_read_2023():
+    domains_and_releases_to_read = [
+        # ("farm_gate_agriculture_energy", "2023-12-13"),
+        # ("farm_gate_emissions_crops", "2023-11-09"),
+        # ("farm_gate_livestock", "2023-11-09"),
+        # ("land_use_drained_organic_soils", "2023-11-09"),
+        # ("land_use_fires", "2023-11-09"),
+        # ("land_use_forests", "2023-11-09"),
+        # ("pre_post_agricultural_production", "2023-11-09"),
+        ("farm_gate_agriculture_energy", "2024-11-14"),
+        ("farm_gate_emissions_crops", "2024-11-14"),
+        ("farm_gate_livestock", "2024-11-14"),
+        ("land_use_drained_organic_soils", "2024-11-14"),
+        ("land_use_fires", "2024-11-14"),
+        ("land_use_forests", "2024-11-14"),
+        ("pre_post_agricultural_production", "2024-11-14"),
+    ]
+
+    read_data(
+        domains_and_releases_to_read=domains_and_releases_to_read,
+        read_path=downloaded_data_path,
+        save_path=extracted_data_path,
+    )
+
+
 def test_yaml_to_python():
     cat = cc.from_yaml("FAO.yaml")
     cat.to_python("FAO.py")
@@ -121,7 +147,7 @@ def test_python_to_yaml():
     assert cat
 
 
-def test_make_dict_comprehension_for_faster_typing():  # noqa: PLR0912 PLR0915
+def test_fao_categorisation():  # noqa: PLR0912 PLR0915
     spec = {
         "name": "FAOSTAT",
         "title": (
@@ -139,6 +165,7 @@ def test_make_dict_comprehension_for_faster_typing():  # noqa: PLR0912 PLR0915
     }
 
     categories = {}
+
     # 0. main categories
     categories["0"] = {
         "title": "Total",
@@ -148,10 +175,7 @@ def test_make_dict_comprehension_for_faster_typing():  # noqa: PLR0912 PLR0915
     children_1 = ["1.A", "1.B"]
     children_2 = ["2.A", "2.B", "2.C", "2.D", "2.E"]
     children_3 = [f"3.{i}" for i in "ABCDEFGHIJKLMNOPQR"]
-    # children_4 = ["4.A"]
-    # children_5 = ["5.A", "5.B"]
-    # children_6 = ["6.A", "6.B", "6.C"]
-    # children_7 = [f"3.{i}" for i in "ABCDEFGHIJKLM"]
+
     main_categories = (
         # category code, name and comment, gases, children
         ("1", "Crops", ["CH4", "N2O"], children_1),
@@ -697,4 +721,3 @@ def test_make_dict_comprehension_for_faster_typing():  # noqa: PLR0912 PLR0915
     # run print(fao_cats.show_as_tree())
     fao_cats.to_python("FAO.py")
     fao_cats.to_yaml("FAO.yaml")
-    pass