From 6757d5b8425c5e574324f4374240d2644d0f3210 Mon Sep 17 00:00:00 2001 From: LSaldyt Date: Sun, 19 Nov 2017 11:25:37 -0700 Subject: [PATCH] Adds cross-chi2 results --- copycat/statistics.py | 25 +++- cross_compare.py | 2 +- distributions/.adj-tests | Bin 883 -> 884 bytes results.txt | 292 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 311 insertions(+), 8 deletions(-) create mode 100644 results.txt diff --git a/copycat/statistics.py b/copycat/statistics.py index 0f8a74c..c085943 100644 --- a/copycat/statistics.py +++ b/copycat/statistics.py @@ -31,24 +31,28 @@ def chi_squared(actual, expected): chiSquared += (O - E) ** 2 / E return degreesFreedom, chiSquared -def chi_squared_test(actual, expected): +def chi_squared_test(actual, expected, show=True): df, chiSquared = chi_squared(actual, expected) if chiSquared >= _chiSquared_table[df]: - print('Significant difference between expected and actual answer distributions: \n' + - 'Chi2 value: {} with {} degrees of freedom'.format(chiSquared, df)) + if show: + print('Significant difference between expected and actual answer distributions: \n' + + 'Chi2 value: {} with {} degrees of freedom'.format(chiSquared, df)) return False return True def cross_formula_chi_squared(actualDict, expectedDict): for ka, actual in actualDict.items(): for ke, expected in expectedDict.items(): - print('Comparing {} with {}'.format(ka, ke)) - chi_squared_test(actual, expected) + print('Comparing {} with {}: '.format(ka, ke), end='') + if not chi_squared_test(actual, expected, show=False): + print('Failed.') + else: + print('Succeeded.') def cross_chi_squared(problemSets): - for i, problemSetA in enumerate(problemSets): - for problemSetB in problemSets[i + 1:]: + for i, (a, problemSetA) in enumerate(problemSets): + for b, problemSetB in problemSets[i + 1:]: for problemA in problemSetA: for problemB in problemSetB: if (problemA.initial == problemB.initial and @@ -56,7 +60,14 @@ def cross_chi_squared(problemSets): problemA.target == problemB.target): answersA = problemA.distributions answersB = problemB.distributions + print('-' * 80) + print('\n') + print('{} x {}'.format(a, b)) + print('Problem: {}:{}::{}:_'.format(problemA.initial, + problemA.modified, + problemA.target)) cross_formula_chi_squared(answersA, answersB) + print('\n') def iso_chi_squared(actualDict, expectedDict): for key in expectedDict.keys(): diff --git a/cross_compare.py b/cross_compare.py index 734c86c..500d2ec 100755 --- a/cross_compare.py +++ b/cross_compare.py @@ -15,7 +15,7 @@ def main(args): with open(filename, 'rb') as infile: pSet = pickle.load(infile) branchProblemSets[filename] = pSet - problemSets.append(pSet) + problemSets.append((filename, pSet)) cross_chi_squared(problemSets) return 0 diff --git a/distributions/.adj-tests b/distributions/.adj-tests index 3f59c8ff05e0c01763f9795f5d1bb4cf3bf80e2e..369d7eba439f987b4a5539ea2919e37b70bc5d3a 100644 GIT binary patch delta 9 Qcmey&_JwUjF*73<02Ji|B>(^b delta 7 Ocmeyu_L*%%F*5)Trvn-Q diff --git a/results.txt b/results.txt new file mode 100644 index 0000000..34cb5ef --- /dev/null +++ b/results.txt @@ -0,0 +1,292 @@ +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.adj-tests +Problem: abc:abd::efg:_ +Comparing None with None: Warning! Expected 0 counts of ffg, but got 1 +Warning! Expected 0 counts of dfg, but got 2 +Warning! Expected 0 counts of efg, but got 1 +Warning! Expected 0 counts of eff, but got 6 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.adj-tests +Problem: abc:abd::ijk:_ +Comparing None with None: Warning! Expected 0 counts of jjk, but got 4 +Warning! Expected 0 counts of ijj, but got 4 +Warning! Expected 0 counts of hjk, but got 1 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.adj-tests +Problem: abc:abd::xyz:_ +Comparing None with None: Warning! Expected 0 counts of xd, but got 1 +Warning! Expected 0 counts of xyz, but got 5 +Warning! Expected 0 counts of dyz, but got 1 +Warning! Expected 0 counts of yyz, but got 9 +Warning! Expected 0 counts of xyy, but got 5 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.adj-tests +Problem: abc:abd::ijkk:_ +Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18 +Warning! Expected 0 counts of ijkd, but got 1 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.adj-tests +Problem: abc:abd::mrrjjj:_ +Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 8 +Warning! Expected 0 counts of mrrjjd, but got 1 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.nuke-temp +Problem: abc:abd::efg:_ +Comparing None with None: Warning! Expected 0 counts of dfg, but got 2 +Warning! Expected 0 counts of efg, but got 1 +Warning! Expected 0 counts of eff, but got 6 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.nuke-temp +Problem: abc:abd::ijk:_ +Comparing None with None: Warning! Expected 0 counts of jjk, but got 4 +Warning! Expected 0 counts of ijj, but got 4 +Warning! Expected 0 counts of hjk, but got 1 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.nuke-temp +Problem: abc:abd::xyz:_ +Comparing None with None: Warning! Expected 0 counts of xd, but got 1 +Warning! Expected 0 counts of xyz, but got 5 +Warning! Expected 0 counts of dyz, but got 1 +Warning! Expected 0 counts of yyz, but got 9 +Warning! Expected 0 counts of xyy, but got 5 +Warning! Expected 0 counts of wyz, but got 9 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.nuke-temp +Problem: abc:abd::ijkk:_ +Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.nuke-temp +Problem: abc:abd::mrrjjj:_ +Comparing None with None: Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.soft-remove +Problem: abc:abd::efg:_ +Comparing None with None: Warning! Expected 0 counts of ffg, but got 1 +Warning! Expected 0 counts of dfg, but got 2 +Warning! Expected 0 counts of efg, but got 1 +Warning! Expected 0 counts of eff, but got 6 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.soft-remove +Problem: abc:abd::ijk:_ +Comparing None with None: Warning! Expected 0 counts of jjk, but got 4 +Warning! Expected 0 counts of ijj, but got 4 +Warning! Expected 0 counts of hjk, but got 1 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.soft-remove +Problem: abc:abd::xyz:_ +Comparing None with None: Warning! Expected 0 counts of xd, but got 1 +Warning! Expected 0 counts of xyz, but got 5 +Warning! Expected 0 counts of dyz, but got 1 +Warning! Expected 0 counts of xyy, but got 5 +Warning! Expected 0 counts of wyz, but got 9 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.soft-remove +Problem: abc:abd::ijkk:_ +Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.legacy x distributions/.soft-remove +Problem: abc:abd::mrrjjj:_ +Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 8 +Warning! Expected 0 counts of mrrjjd, but got 1 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.nuke-temp +Problem: abc:abd::efg:_ +Comparing None with None: Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.nuke-temp +Problem: abc:abd::ijk:_ +Comparing None with None: Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.nuke-temp +Problem: abc:abd::xyz:_ +Comparing None with None: Warning! Expected 0 counts of wyz, but got 1 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.nuke-temp +Problem: abc:abd::ijkk:_ +Comparing None with None: Warning! Expected 0 counts of ijd, but got 1 +Warning! Expected 0 counts of ijll, but got 17 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.nuke-temp +Problem: abc:abd::mrrjjj:_ +Comparing None with None: Warning! Expected 0 counts of mrrkkk, but got 26 +Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.soft-remove +Problem: abc:abd::efg:_ +Comparing None with None: Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.soft-remove +Problem: abc:abd::ijk:_ +Comparing None with None: Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.soft-remove +Problem: abc:abd::xyz:_ +Comparing None with None: Warning! Expected 0 counts of wyz, but got 1 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.soft-remove +Problem: abc:abd::ijkk:_ +Comparing None with None: Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.adj-tests x distributions/.soft-remove +Problem: abc:abd::mrrjjj:_ +Comparing None with None: Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.nuke-temp x distributions/.soft-remove +Problem: abc:abd::efg:_ +Comparing None with None: Warning! Expected 0 counts of ffg, but got 1 +Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.nuke-temp x distributions/.soft-remove +Problem: abc:abd::ijk:_ +Comparing None with None: Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.nuke-temp x distributions/.soft-remove +Problem: abc:abd::xyz:_ +Comparing None with None: Succeeded. + + +-------------------------------------------------------------------------------- + + +distributions/.nuke-temp x distributions/.soft-remove +Problem: abc:abd::ijkk:_ +Comparing None with None: Failed. + + +-------------------------------------------------------------------------------- + + +distributions/.nuke-temp x distributions/.soft-remove +Problem: abc:abd::mrrjjj:_ +Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 1 +Warning! Expected 0 counts of mrrjjd, but got 2 +Failed. + +