Finish notebook eval.

4 files changed, 155 insertions, 72 deletions

diff --git a/re/epa.ipynb b/re/epa.ipynb
index 7e329b2..654e7ad 100644
--- a/re/epa.ipynb
+++ b/re/epa.ipynb
@@ -494,11 +494,63 @@
    ]
   },
   {
+   "cell_type": "markdown",
+   "id": "1248a648-e70f-433c-91b6-5449b70b6516",
+   "metadata": {},
+   "source": [
+    "### Miscellaneous"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6cca69c8-48a8-4f82-bba0-35f9df7c440e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "param_categories = {\n",
+    "    \"a=-1\": [\"projective-1\"],\n",
+    "    \"a=-3\": [\"projective-3\", \"jacobian-3\", \"xyzz-3\"],\n",
+    "    \"a=0\": [\"jacobian-0\"],\n",
+    "    \"generic\": [\"jacobian\", \"projective\", \"modified\", \"xyzz\", \"xz\"],\n",
+    "    \"b=0\": [\"w12-0\"]\n",
+    "}\n",
+    "cfg_categories = {}\n",
+    "for name, coord_names in param_categories.items():\n",
+    "    category_cfgs = set()\n",
+    "    for coord_name in coord_names:\n",
+    "        coords = model.coordinates[coord_name]\n",
+    "        category_cfgs.update(filter(lambda cfg: cfg[0].coordinate_model == coords and cfg[1].coordinate_model == coords, configs))\n",
+    "    cfg_categories[name] = category_cfgs\n",
+    "category_map = {cfg: {\"category\": name} for name, category_cfgs in cfg_categories.items() for cfg in category_cfgs}\n",
+    "dmap_categories = Map.from_io_maps(configs, category_map)"
+   ]
+  },
+  {
    "cell_type": "code",
    "execution_count": null,
    "id": "6388a793-5433-4815-960e-7b2cc0f2211a",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "dmap = Map.from_sets(configs, precomp, deduplicate=True)\n",
+    "tree = Tree.build(configs, dmap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a975e6eb-51ea-4221-b2c6-c86ac8eb1739",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(tree.describe())"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "71501ace-e964-48ad-b8fd-9859363ed28d",
+   "metadata": {},
    "source": []
   }
  ],
diff --git a/re/eval.py b/re/eval.py
index 6086c04..1065af1 100644
--- a/re/eval.py
+++ b/re/eval.py
@@ -295,6 +295,7 @@ def _plot_symmetric(rate, cmap, name, unit, xticks, xlabel, yticks, ylabel, colo
     im = ax.imshow(rate.T, cmap=cmap, origin="lower", vmin=vmin, vmax=vmax)
     cbar_ax = fig.add_axes((0.85, 0.15, 0.04, 0.69))
     cbar = fig.colorbar(im, cax=cbar_ax)
+    cbar.ax.yaxis.set_label_coords(2.5, 0.5);
     cbar.ax.set_ylabel(name, rotation=-90, va="bottom")
     if baseline:
         cbar.ax.axhline(baseline, color="red", linestyle="--")
@@ -377,6 +378,7 @@ def _plot_asymmetric(rate, cmap, name, unit, color_threshold, vmin=None, vmax=No
     fig.tight_layout(h_pad=1.5, rect=(0, 0, 0.9, 1))
     cbar_ax = fig.add_axes((0.9, 0.10, 0.02, 0.84))
     cbar = fig.colorbar(im, cax=cbar_ax)
+    cbar.ax.yaxis.set_label_coords(2.5, 0.5);
     cbar.ax.set_ylabel(name, rotation=-90, va="bottom")
     if baseline:
         cbar.ax.axhline(baseline, color="red", linestyle="--")
@@ -414,7 +416,7 @@ def success_rate_vs_majority_asymmetric(correct_rate_b):
         ax.plot(majs, crs, label=f"total_error = {total_err}")
     ax.set_xticks(majs)
     ax.set_xlabel("majority")
-    ax.set_ylabel("success rate")
+    ax.set_ylabel("success rate (%)")
     ax.legend(bbox_to_anchor=(1, 1.02))
     fig.tight_layout()
     return fig
@@ -434,3 +436,15 @@ def amount_rate_binomial(amount_rate):
 
 def query_rate_binomial(query_rate):
     return _plot_symmetric(query_rate, mako, "Oracle query rate", "", nums, "binom n", smpls, "samples", 0.5)
+
+
+def store(path, correct_rate, precise_rate, amount_rate, query_rate):
+    vs = {"correct_rate": correct_rate, "precise_rate": precise_rate, "amount_rate": amount_rate, "query_rate": query_rate}
+    ds = xr.Dataset(data_vars=vs)
+    ds.to_netcdf(path)
+
+
+def load(path):
+    ds = xr.open_dataset(path)
+    return ds.correct_rate, ds.precise_rate, ds.amount_rate, ds.query_rate
+
diff --git a/re/rpa.ipynb b/re/rpa.ipynb
index f0fa1d6..3a1f2d4 100644
--- a/re/rpa.ipynb
+++ b/re/rpa.ipynb
@@ -61,8 +61,9 @@
     "                    success_rate_symmetric, success_rate_asymmetric,\n",
     "                    query_rate_symmetric, query_rate_asymmetric,\n",
     "                    precise_rate_symmetric, precise_rate_asymmetric,\n",
+    "                    amount_rate_symmetric, amount_rate_asymmetric,\n",
     "                    success_rate_vs_majority_symmetric, success_rate_vs_majority_asymmetric,\n",
-    "                    success_rate_vs_query_rate_symmetric)"
+    "                    success_rate_vs_query_rate_symmetric, load, store)"
    ]
   },
   {
@@ -471,27 +472,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "srs_fig = success_rate_symmetric(correct_rate, 100 / len(multipliers))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "131f2bee-4c07-4449-a380-06082861d753",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "qrs_fig = query_rate_symmetric(query_rate)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "901bfdaa-5767-4c7b-92d5-9f82dd075ea6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "prs_fig = precise_rate_symmetric(precise_rate)"
+    "success_rate_symmetric(correct_rate, 100 / len(multipliers)).savefig(\"rpa_re_success_rate_symmetric.pdf\", bbox_inches=\"tight\")\n",
+    "query_rate_symmetric(query_rate).savefig(\"rpa_re_query_rate_symmetric.pdf\", bbox_inches=\"tight\")\n",
+    "precise_rate_symmetric(precise_rate).savefig(\"rpa_re_precise_rate_symmetric.pdf\", bbox_inches=\"tight\")\n",
+    "amount_rate_symmetric(amount_rate).savefig(\"rpa_re_amount_rate_symmetric.pdf\", bbox_inches=\"tight\")"
    ]
   },
   {
@@ -509,37 +493,36 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "srqrs_fig = success_rate_vs_query_rate_symmetric(query_rate, correct_rate)"
+    "success_rate_vs_query_rate_symmetric(query_rate, correct_rate).savefig(\"rpa_re_scatter_symmetric.pdf\", bbox_inches=\"tight\")\n",
+    "success_rate_vs_majority_symmetric(correct_rate).savefig(\"rpa_re_plot_symmetric.pdf\", bbox_inches=\"tight\")"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c2cfdca4-7e88-4138-b01f-696ecdcfd2bd",
+   "cell_type": "markdown",
+   "id": "8c747434-84bb-4acd-a994-45a4b4859a6e",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "srms_fig = success_rate_vs_majority_symmetric(correct_rate)"
+    "And save the results for later."
    ]
   },
   {
-   "cell_type": "markdown",
-   "id": "8c747434-84bb-4acd-a994-45a4b4859a6e",
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "857f39ad-f6ba-4006-8da9-92f4318819e2",
    "metadata": {},
+   "outputs": [],
    "source": [
-    "And save the results for later."
+    "store(\"rpa_re_symmetric.nc\", correct_rate, precise_rate, amount_rate, query_rate)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "857f39ad-f6ba-4006-8da9-92f4318819e2",
+   "id": "efb26038-87ea-4a5d-8682-f855c2bec13f",
    "metadata": {},
    "outputs": [],
    "source": [
-    "np.save(\"rpa_re_correct_rate\", correct_rate)\n",
-    "np.save(\"rpa_re_precise_rate\", precise_rate)\n",
-    "np.save(\"rpa_re_query_rate\", query_rate)"
+    "correct_rate, precise_rate, amount_rate, query_rate = load(\"rpa_re_symmetric.nc\")"
    ]
   },
   {
@@ -574,57 +557,39 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sra_fig = success_rate_asymmetric(correct_rate_b, 100 / len(multipliers))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6b14a5eb-cfde-4e3e-a4f3-0d21dcca2dff",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "qra_fig = query_rate_asymmetric(query_rate_b)"
+    "success_rate_asymmetric(correct_rate_b, 100 / len(multipliers)).savefig(\"rpa_re_success_rate_asymmetric.pdf\", bbox_inches=\"tight\")\n",
+    "query_rate_asymmetric(query_rate_b).savefig(\"rpa_re_query_rate_asymmetric.pdf\", bbox_inches=\"tight\")\n",
+    "precise_rate_asymmetric(precise_rate_b).savefig(\"rpa_re_precise_rate_asymmetric.pdf\", bbox_inches=\"tight\")\n",
+    "amount_rate_asymmetric(amount_rate_b).savefig(\"rpa_re_amount_rate_asymmetric.pdf\", bbox_inches=\"tight\")\n",
+    "success_rate_vs_majority_asymmetric(correct_rate_b).savefig(\"rpa_re_plot_asymmetric.pdf\", bbox_inches=\"tight\")"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c5c7fe0e-88be-4d4e-a8fb-352ea85060f1",
+   "cell_type": "markdown",
+   "id": "67a6bba8-84e8-48f1-9b3b-b0b8715c71a6",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "pra_fig = precise_rate_asymmetric(precise_rate_b)"
+    "And save the results for later."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "e4e999bf-bdd0-4b9f-86c6-1fdee773fd27",
+   "id": "85933a5e-f526-4002-9203-0e4ae4f731d0",
    "metadata": {},
    "outputs": [],
    "source": [
-    "srma_fig = success_rate_vs_majority_asymmetric(correct_rate_b)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "67a6bba8-84e8-48f1-9b3b-b0b8715c71a6",
-   "metadata": {},
-   "source": [
-    "And save the results for later."
+    "store(\"rpa_re_asymmetric.nc\", correct_rate_b, precise_rate_b, amount_rate_b, query_rate_b)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "85933a5e-f526-4002-9203-0e4ae4f731d0",
+   "id": "ddde34f8-7cde-449d-a2e3-289e1aefba72",
    "metadata": {},
    "outputs": [],
    "source": [
-    "np.save(\"rpa_re_correct_rate_b\", correct_rate_b)\n",
-    "np.save(\"rpa_re_precise_rate_b\", precise_rate_b)\n",
-    "np.save(\"rpa_re_query_rate_b\", query_rate_b)"
+    "correct_rate_b, precise_rate_b, amount_rate_b, query_rate_b = load(\"rpa_re_asymmetric.nc\")"
    ]
   },
   {
@@ -726,7 +691,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9dc922b4-3123-489f-98e2-4c20f1d65816",
+   "id": "7d974e88-77fb-4331-9477-25e6d18dc6ba",
    "metadata": {},
    "outputs": [],
    "source": []
diff --git a/re/zvp.ipynb b/re/zvp.ipynb
index 500d0fe..9d90371 100644
--- a/re/zvp.ipynb
+++ b/re/zvp.ipynb
@@ -32,6 +32,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import holoviews as hv\n",
+    "import xarray as xr\n",
     "import random\n",
     "import tabulate\n",
     "import pickle\n",
@@ -74,7 +75,7 @@
     "                    amount_rate_symmetric, amount_rate_asymmetric, amount_rate_binomial,\n",
     "                    precise_rate_symmetric, precise_rate_asymmetric, precise_rate_binomial,\n",
     "                    success_rate_vs_majority_symmetric, success_rate_vs_majority_asymmetric,\n",
-    "                    success_rate_vs_query_rate_symmetric)\n",
+    "                    success_rate_vs_query_rate_symmetric, load, store)\n",
     "\n",
     "\n",
     "# Allow to use \"spawn\" multiprocessing method for function defined in a Jupyter notebook.\n",
@@ -570,7 +571,10 @@
    "outputs": [],
    "source": [
     "with open(\"all_points.pickle\", \"rb\") as f:\n",
-    "    all_points, all_points_filtered = pickle.load(f)"
+    "    all_points, all_points_filtered = pickle.load(f)\n",
+    "    \n",
+    "print(f\"Got {len(all_points)} points.\")\n",
+    "print(f\"Got {len(all_points_filtered)} filtered points\")"
    ]
   },
   {
@@ -949,7 +953,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "np.savez(\"zvp_re_symmetric\", correct_rate=correct_rate, precise_rate=precise_rate, amount_rate=amount_rate, query_rate=query_rate)"
+    "store(\"zvp_re_symmetric.nc\", correct_rate, precise_rate, amount_rate, query_rate)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a443d93-cf71-4c6e-bc11-998a31a47656",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "correct_rate, precise_rate, amount_rate, query_rate = load(\"zvp_re_symmetric.nc\")"
    ]
   },
   {
@@ -984,7 +998,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "np.savez(\"zvp_re_asymmetric\", correct_rate=correct_rate_b, precise_rate=precise_rate_b, amount_rate=amount_rate_b, query_rate=query_rate_b)"
+    "store(\"zvp_re_asymmetric.nc\", correct_rate_b, precise_rate_b, amount_rate_b, query_rate_b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8924c80d-2d65-40a6-b589-85bbe93d1094",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "correct_rate_b, precise_rate_b, amount_rate_b, query_rate_b = load(\"zvp_re_asymmetric.nc\")"
    ]
   },
   {
@@ -1018,7 +1042,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "np.savez(\"zvp_re_binomial\", correct_rate=correct_rate_c, precise_rate=precise_rate_c, amount_rate=amount_rate_c, query_rate=query_rate_c)"
+    "store(\"zvp_re_binomial.nc\", correct_rate_c, precise_rate_c, amount_rate_c, query_rate_c)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "088a9776-0713-402d-b351-994ba2e24d4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "correct_rate_c, precise_rate_c, amount_rate_c, query_rate_c = load(\"zvp_re_binomial.nc\")"
    ]
   },
   {
@@ -1357,6 +1391,24 @@
    "id": "f98fca35-5c01-434d-95b5-0ba81ab37721",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "for tree in (tree_remapped, tree_count, tree_position):\n",
+    "    same = 0\n",
+    "    for leaf in tree.leaves:\n",
+    "        cds = set()\n",
+    "        for c in leaf.cfgs:\n",
+    "            cds.add(c[0].coordinate_model)\n",
+    "            cds.add(c[1].coordinate_model)\n",
+    "        same += len(cds) == 1\n",
+    "    print(same / len(tree.leaves))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "af01658f-6c95-483c-9d8d-ce6f34566cee",
+   "metadata": {},
+   "outputs": [],
    "source": []
   }
  ],