[DATALAD] Recorded changes

conv_FAO_IPPCC2006_PRIMAP_N2O.csv

@@ -1,8 +1,11 @@
 # references: N2O emissions only
 # last_update: 2024-10-14
 FAO,IPCC2006_PRIMAP,comment
-M.3.MM, 3.A.2, Manure management
-1.A.1.b + 1.A.2.b + 1.A.6.b + 1.A.7.b, 3.C.1.b, FAO 1.A.X.b Burning of crop residues to IPCC 3.C.1.b Biomass Burning In Croplands
+M.3.MM, 3.A.2, FAO All animals manure management to IPCC 3.A.2 Manure Management
+M.1.BCR, 3.C.1.b, All Crops - Burning crop residues (Emissions N2O) (or the sum of FAO 1.A.X.b) to IPCC 3.C.1.b Biomass Burning In Croplands
+M.3.MP, M.3.C.MP, FAO Manure left on pasture to IPCC Direct and indirect emissions from manure left on pasture
+M.3.MA, M.3.C.MA, FAO Manure applied to soils to IPCC Direct and indirect emissions from manure applied to soils
+M.1.CR, M.3.C.CR, FAO Crop residues to IPCC Direct and indirect emissions from crop residues
 M.3.MP + M.3.MA + M.1.CR, 3.C.4
 1.B.1,M.3.C.4.SF,Better to map indivudually in case it covers different range of years
 1.B.2.a + 1.B.2.b,M.3.C.5.SF,Better to map individually in case it covers different range of years

extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13.csv

@@ -1 +1 @@
-../../.git/annex/objects/GJ/Z3/MD5E-s6065549--e0ce12dad1da8713b5b03b48321a54b8.csv/MD5E-s6065549--e0ce12dad1da8713b5b03b48321a54b8.csv
+../../.git/annex/objects/wx/VV/MD5E-s5993308--a10486471aeb45e4adce1d8ee63741e0.csv/MD5E-s5993308--a10486471aeb45e4adce1d8ee63741e0.csv

extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13.nc

@@ -1 +1 @@
-../../.git/annex/objects/jm/MK/MD5E-s1728952--5e26bf3a64fcda4e4e62073d1ffee66c.nc/MD5E-s1728952--5e26bf3a64fcda4e4e62073d1ffee66c.nc
+../../.git/annex/objects/z0/fv/MD5E-s1873879--acc086be8a16a483c7ab3a8d9a9c4658.nc/MD5E-s1873879--acc086be8a16a483c7ab3a8d9a9c4658.nc

extracted_data/v2023-12-13/FAOSTAT_Agrifood_system_emissions_v2023-12-13_raw.nc

@@ -1 +1 @@
-../../.git/annex/objects/zM/X2/MD5E-s14597532--9ab0a8c13b665135e8876baf957da33f.nc/MD5E-s14597532--9ab0a8c13b665135e8876baf957da33f.nc
+../../.git/annex/objects/q4/Q3/MD5E-s13978443--ecbdc2bf619cb39d0512b8717792edc5.nc/MD5E-s13978443--ecbdc2bf619cb39d0512b8717792edc5.nc

extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14.csv

@@ -1 +1 @@
-../../.git/annex/objects/3z/XQ/MD5E-s6190399--16a342c6e944090d4d630915a7e6e4ff.csv/MD5E-s6190399--16a342c6e944090d4d630915a7e6e4ff.csv
+../../.git/annex/objects/Jm/87/MD5E-s6117436--56ec6d78760e9af0d835a6d5395c0a1d.csv/MD5E-s6117436--56ec6d78760e9af0d835a6d5395c0a1d.csv

extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14.nc

@@ -1 +1 @@
-../../.git/annex/objects/60/V5/MD5E-s1753492--894ac3cc25f273e8b476269a8ef0d164.nc/MD5E-s1753492--894ac3cc25f273e8b476269a8ef0d164.nc
+../../.git/annex/objects/p3/gw/MD5E-s1905112--d05e0d4cbff075e33d802848beb16ea2.nc/MD5E-s1905112--d05e0d4cbff075e33d802848beb16ea2.nc

extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.csv

@@ -1 +1 @@
-../../.git/annex/objects/xp/61/MD5E-s34032099--9dfa52731d2cd7d035ea8aafa2a8f650.csv/MD5E-s34032099--9dfa52731d2cd7d035ea8aafa2a8f650.csv
+../../.git/annex/objects/vp/pV/MD5E-s34289102--5198b88b05708f715b0db35ed579d6af.csv/MD5E-s34289102--5198b88b05708f715b0db35ed579d6af.csv

extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14_raw.nc

@@ -1 +1 @@
-../../.git/annex/objects/Jz/xQ/MD5E-s15356164--07fe672ce4b695d7cef28b8740d5fa88.nc/MD5E-s15356164--07fe672ce4b695d7cef28b8740d5fa88.nc
+../../.git/annex/objects/68/q4/MD5E-s15217110--49548ca960988719cc62fe57b9e45ce1.nc/MD5E-s15217110--49548ca960988719cc62fe57b9e45ce1.nc

src/faostat_data_primap/helper/category_aggregation.py

@@ -6,6 +6,15 @@ Definitions for category aggregation.
 # There are discrepancies of up to 100% due to rounding errors for small values,
 # for example, 0.0001 (rounded from 0.00006) + 0.0004 (rounded from 0.00036)
 # = 0.00042 which is then rounded to 0.0004, while the consistency check expects 0.0005
+# There are even more extreme example, where we need a tolerance of 100%:
+# Eswatini, 1976:
+# 1.A.1.a Crop residues (emissions N2O) = 0.0001
+# 1.A.1.a.i Crop residues (Indirect emissions N2O) = 0
+# 1.A.1.a.ii Crop residues (Direct emissions N2O) = 0
+# Our way to deal with it, was to set the tolerance to 1% and look at the
+# countries / sectors that yielded an error. If only a few countries and years
+# are affected, it is likely just a rounding error. If all years are affected
+# there may be something wrong with the data
 agg_info_fao = {
     "category (FAO)": {
         "1.A.1.a": {  # wheat
@@ -397,6 +406,9 @@ agg_info_ipcc2006_primap_N2O = {
                 "3.C.1.b",  # Biomass Burning In Croplands - looks good (CH4, N2O)
                 "3.C.1.c",  # Biomass Burning in Grasslands - looks good (CH4)
                 "3.C.4",  # Direct N2O Emissions from Managed Soils
+                # "M.3.C.MP",
+                # "M.3.C.MA",
+                # "M.3.C.CR",
                 "M.3.C.4.SF",  # synthetic fertilisers direct
                 # "3.C.5",  # Indirect N2O Emissions from Managed Soils, currently empty
                 "M.3.C.5.SF",  # synthetic fertilisers indirect
@@ -415,6 +427,9 @@ agg_info_ipcc2006_primap_N2O = {
             "sources": [
                 "M.3.C.1.AG",  # TODO 3.C.1 would be correct, but doesn't match 2023
                 "3.C.4",  # Direct N2O Emissions from Managed Soils
+                # "M.3.C.MP",
+                # "M.3.C.MA",
+                # "M.3.C.CR",
                 "M.3.C.4.SF",  # synthetic fertilisers direct
                 # "3.C.5",  # Indirect N2O Emissions from Managed Soils, empty
                 "M.3.C.5.SF",  # synthetic fertilisers indirect

src/faostat_data_primap/helper/definitions.py

@@ -424,8 +424,8 @@ read_config_all: Any = {
             "items-elements_to_remove": [
                 "All Crops - Crop residues (Direct emissions N2O)",
                 "All Crops - Crop residues (Indirect emissions N2O)",
-                "All Crops - Burning crop residues (Emissions N2O)",
-                "All Crops - Burning crop residues (Emissions CH4)",
+                # "All Crops - Burning crop residues (Emissions N2O)",
+                # "All Crops - Burning crop residues (Emissions CH4)",
             ],
             "columns_to_drop": [
                 "Element",

tests/unit/test_conversion.py

@@ -36,6 +36,8 @@ def test_processed_output_remains_the_same():
     ]
 
     # compare
+    # xr.testing.assert_allclose(ds_processed, ds_processed_new, rtol=1e-5, atol=1e-8)
+
     assert ds_processed.broadcast_equals(ds_processed_new)