Adds cross-chi2 results

copycat/statistics.py

@@ -31,24 +31,28 @@ def chi_squared(actual, expected):
             chiSquared += (O - E) ** 2 / E
     return degreesFreedom, chiSquared
 
-def chi_squared_test(actual, expected):
+def chi_squared_test(actual, expected, show=True):
     df, chiSquared = chi_squared(actual, expected)
 
     if chiSquared >= _chiSquared_table[df]:
-        print('Significant difference between expected and actual answer distributions: \n' +
-            'Chi2 value: {} with {} degrees of freedom'.format(chiSquared, df))
+        if show:
+            print('Significant difference between expected and actual answer distributions: \n' +
+                'Chi2 value: {} with {} degrees of freedom'.format(chiSquared, df))
         return False
     return True
 
 def cross_formula_chi_squared(actualDict, expectedDict):
     for ka, actual in actualDict.items():
         for ke, expected in expectedDict.items():
-            print('Comparing {} with {}'.format(ka, ke))
-            chi_squared_test(actual, expected)
+            print('Comparing {} with {}: '.format(ka, ke), end='')
+            if not chi_squared_test(actual, expected, show=False):
+                print('Failed.')
+            else:
+                print('Succeeded.')
 
 def cross_chi_squared(problemSets):
-    for i, problemSetA in enumerate(problemSets):
-        for problemSetB in problemSets[i + 1:]:
+    for i, (a, problemSetA) in enumerate(problemSets):
+        for b, problemSetB in problemSets[i + 1:]:
             for problemA in problemSetA:
                 for problemB in problemSetB:
                     if (problemA.initial  == problemB.initial and 
@@ -56,7 +60,14 @@ def cross_chi_squared(problemSets):
                         problemA.target   == problemB.target):
                         answersA = problemA.distributions
                         answersB = problemB.distributions
+                        print('-' * 80)
+                        print('\n')
+                        print('{} x {}'.format(a, b))
+                        print('Problem:  {}:{}::{}:_'.format(problemA.initial,
+                                                             problemA.modified,
+                                                             problemA.target))
                         cross_formula_chi_squared(answersA, answersB)
+                        print('\n')
 
 def iso_chi_squared(actualDict, expectedDict):
     for key in expectedDict.keys():

cross_compare.py

@@ -15,7 +15,7 @@ def main(args):
         with open(filename, 'rb') as infile:
             pSet = pickle.load(infile)
             branchProblemSets[filename] = pSet
-            problemSets.append(pSet)
+            problemSets.append((filename, pSet))
     cross_chi_squared(problemSets)
     return 0
 

results.txt

@@ -0,0 +1,292 @@
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.adj-tests
+Problem:  abc:abd::efg:_
+Comparing None with None: Warning! Expected 0 counts of ffg, but got 1
+Warning! Expected 0 counts of dfg, but got 2
+Warning! Expected 0 counts of efg, but got 1
+Warning! Expected 0 counts of eff, but got 6
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.adj-tests
+Problem:  abc:abd::ijk:_
+Comparing None with None: Warning! Expected 0 counts of jjk, but got 4
+Warning! Expected 0 counts of ijj, but got 4
+Warning! Expected 0 counts of hjk, but got 1
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.adj-tests
+Problem:  abc:abd::xyz:_
+Comparing None with None: Warning! Expected 0 counts of xd, but got 1
+Warning! Expected 0 counts of xyz, but got 5
+Warning! Expected 0 counts of dyz, but got 1
+Warning! Expected 0 counts of yyz, but got 9
+Warning! Expected 0 counts of xyy, but got 5
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.adj-tests
+Problem:  abc:abd::ijkk:_
+Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18
+Warning! Expected 0 counts of ijkd, but got 1
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.adj-tests
+Problem:  abc:abd::mrrjjj:_
+Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 8
+Warning! Expected 0 counts of mrrjjd, but got 1
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.nuke-temp
+Problem:  abc:abd::efg:_
+Comparing None with None: Warning! Expected 0 counts of dfg, but got 2
+Warning! Expected 0 counts of efg, but got 1
+Warning! Expected 0 counts of eff, but got 6
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.nuke-temp
+Problem:  abc:abd::ijk:_
+Comparing None with None: Warning! Expected 0 counts of jjk, but got 4
+Warning! Expected 0 counts of ijj, but got 4
+Warning! Expected 0 counts of hjk, but got 1
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.nuke-temp
+Problem:  abc:abd::xyz:_
+Comparing None with None: Warning! Expected 0 counts of xd, but got 1
+Warning! Expected 0 counts of xyz, but got 5
+Warning! Expected 0 counts of dyz, but got 1
+Warning! Expected 0 counts of yyz, but got 9
+Warning! Expected 0 counts of xyy, but got 5
+Warning! Expected 0 counts of wyz, but got 9
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.nuke-temp
+Problem:  abc:abd::ijkk:_
+Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.nuke-temp
+Problem:  abc:abd::mrrjjj:_
+Comparing None with None: Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.soft-remove
+Problem:  abc:abd::efg:_
+Comparing None with None: Warning! Expected 0 counts of ffg, but got 1
+Warning! Expected 0 counts of dfg, but got 2
+Warning! Expected 0 counts of efg, but got 1
+Warning! Expected 0 counts of eff, but got 6
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.soft-remove
+Problem:  abc:abd::ijk:_
+Comparing None with None: Warning! Expected 0 counts of jjk, but got 4
+Warning! Expected 0 counts of ijj, but got 4
+Warning! Expected 0 counts of hjk, but got 1
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.soft-remove
+Problem:  abc:abd::xyz:_
+Comparing None with None: Warning! Expected 0 counts of xd, but got 1
+Warning! Expected 0 counts of xyz, but got 5
+Warning! Expected 0 counts of dyz, but got 1
+Warning! Expected 0 counts of xyy, but got 5
+Warning! Expected 0 counts of wyz, but got 9
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.soft-remove
+Problem:  abc:abd::ijkk:_
+Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.legacy x distributions/.soft-remove
+Problem:  abc:abd::mrrjjj:_
+Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 8
+Warning! Expected 0 counts of mrrjjd, but got 1
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.nuke-temp
+Problem:  abc:abd::efg:_
+Comparing None with None: Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.nuke-temp
+Problem:  abc:abd::ijk:_
+Comparing None with None: Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.nuke-temp
+Problem:  abc:abd::xyz:_
+Comparing None with None: Warning! Expected 0 counts of wyz, but got 1
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.nuke-temp
+Problem:  abc:abd::ijkk:_
+Comparing None with None: Warning! Expected 0 counts of ijd, but got 1
+Warning! Expected 0 counts of ijll, but got 17
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.nuke-temp
+Problem:  abc:abd::mrrjjj:_
+Comparing None with None: Warning! Expected 0 counts of mrrkkk, but got 26
+Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.soft-remove
+Problem:  abc:abd::efg:_
+Comparing None with None: Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.soft-remove
+Problem:  abc:abd::ijk:_
+Comparing None with None: Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.soft-remove
+Problem:  abc:abd::xyz:_
+Comparing None with None: Warning! Expected 0 counts of wyz, but got 1
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.soft-remove
+Problem:  abc:abd::ijkk:_
+Comparing None with None: Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.adj-tests x distributions/.soft-remove
+Problem:  abc:abd::mrrjjj:_
+Comparing None with None: Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.nuke-temp x distributions/.soft-remove
+Problem:  abc:abd::efg:_
+Comparing None with None: Warning! Expected 0 counts of ffg, but got 1
+Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.nuke-temp x distributions/.soft-remove
+Problem:  abc:abd::ijk:_
+Comparing None with None: Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.nuke-temp x distributions/.soft-remove
+Problem:  abc:abd::xyz:_
+Comparing None with None: Succeeded.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.nuke-temp x distributions/.soft-remove
+Problem:  abc:abd::ijkk:_
+Comparing None with None: Failed.
+
+
+--------------------------------------------------------------------------------
+
+
+distributions/.nuke-temp x distributions/.soft-remove
+Problem:  abc:abd::mrrjjj:_
+Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 1
+Warning! Expected 0 counts of mrrjjd, but got 2
+Failed.
+
+