Python Scripts for Skewness, Download Weights, Load Weights

scripts/weight_skewness/compute_skewness.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+# This file is part of Leela Chess.
+# Copyright (C) 2018 Brian Konzman
+#
+# Leela Chess is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Leela Chess is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Leela Chess. If not, see <http://www.gnu.org/licenses/>.
+
+
+# This script only works for 15x196 networks
+
+from weights_file import read_weights_file
+from scipy.stats import skew
+from scipy.stats import kurtosis
+import numpy as np
+import os
+
+import matplotlib.pyplot as plt
+
+
+directory = os.getcwd()
+
+data_points = list()
+ids = list()
+
+files = os.listdir(directory)
+for file in files:
+    id = int(file.split('_')[1].split('.')[0])
+    ids.append(id)
+
+ids.sort()
+
+data_points = list()
+policy_skews = list()
+value_skews = list()
+ids_used = list()
+first_conv_inputs_skews = list()
+second_conv_inputs_skews = list()
+middle_conv_inputs_skews = list()
+last_conv_inputs_skews = list()
+
+FIRST_15x196_ID = 227
+POLICY_FCL_INDEX = 128
+
+for id_number in ids:
+    filename = 'weights_' + str(id_number) + '.txt.gz'
+    if filename.endswith(".gz") and id_number >= FIRST_15x196_ID:
+        file = os.path.join(directory, filename)
+        filters, blocks, weights = read_weights_file(file)
+
+        if type(weights) == list:
+            policy_skews.append(skew(weights[POLICY_FCL_INDEX]))
+
+            ids_used.append(id_number)
+            print('ID' + str(id_number) + ' complete')
+        else:
+            print(type(weights))
+            print(str(id_number))
+
+        continue
+    else:
+        continue
+
+plt.plot(ids_used, policy_skews)
+
+plt.show()

scripts/weight_skewness/download_weights.py

@@ -0,0 +1,57 @@
+# This file is part of Leela Chess.
+# Copyright (C) 2018 github username so-much-meta
+# Copyright (C) 2018 Brian Konzman
+#
+# Leela Chess is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Leela Chess is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Leela Chess. If not, see <http://www.gnu.org/licenses/>.
+
+# Downloads all weights files from lczero.org, does not try to download files already present in destination folder
+
+from bs4 import BeautifulSoup
+import os
+import requests
+import shutil
+
+page = requests.get("http://lczero.org/networks")
+soup = BeautifulSoup(page.content, 'html.parser')
+network_table = soup.find_all('tr')
+
+download_links = dict()
+
+for i in range(1, len(network_table)):
+    row = list(network_table[i])
+    net_number = str(row[1]).split('>')[1].split('<')[0]
+    download_links[net_number] = 'http://lczero.org' + str(row[3]).split('href="')[1].split('"')[0]
+
+most_recent_net_number = str(list(network_table[1])[1]).split('>')[1].split('<')[0]
+
+directory = os.getcwd()
+all_weights = list()
+id_numbers = list()
+
+# remove links that we have already downloaded
+for filename in os.listdir(directory):
+    if filename.endswith(".gz"):
+        net_id = filename.split('_')[1].split('.')[0]
+        if net_id in download_links.keys():
+            download_links.pop(net_id)
+
+
+keys = download_links.keys()
+for k in keys:
+    link = download_links.get(k)
+    r = requests.get(link, stream=True)
+    if r.status_code == 200:
+        with open(directory + 'weights_' + k + '.txt.gz', 'wb') as f:
+            r.raw.decode_content = True
+            shutil.copyfileobj(r.raw, f)

scripts/weight_skewness/weights_file.py

@@ -0,0 +1,44 @@
+# This file is part of Leela Chess.
+# Copyright (C) 2018 Brian Konzman
+#
+# Leela Chess is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Leela Chess is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Leela Chess. If not, see <http://www.gnu.org/licenses/>.
+
+# Only supports weights version 2
+
+import gzip
+
+LEELA_WEIGHTS_VERSION = '2'
+
+
+def read_weights_file(filename):
+    if '.gz' in filename:
+        opener = gzip.open
+    else:
+        opener = open
+    with opener(filename, 'r') as f:
+        version = f.readline().decode('ascii')
+        if version != '{}\n'.format(LEELA_WEIGHTS_VERSION):
+            raise ValueError("Invalid version {}".format(version.strip()))
+        weights = []
+        for e, line in enumerate(f):
+            line = line.decode('ascii')
+            weight = list(map(float, line.split(' ')))
+            weights.append(weight)
+            if e == 1:
+                filters = len(line.split(' '))
+        blocks = e - (3 + 14)
+        if blocks % 8 != 0:
+            raise ValueError("Inconsistent number of weights in the file")
+        blocks //= 8
+    return (filters, blocks, weights)