alex/copycat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adds cross-chi2 results

Browse Source
This commit is contained in:
LSaldyt
2017-11-19 11:25:37 -07:00
parent 402e66409a
commit 6757d5b842
4 changed files with 311 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
copycat/statistics.py
View File

@ -31,24 +31,28 @@ def chi_squared(actual, expected):
chiSquared += (O - E) ** 2 / E chiSquared += (O - E) ** 2 / E
return degreesFreedom, chiSquared return degreesFreedom, chiSquared
def chi_squared_test(actual, expected): def chi_squared_test(actual, expected, show=True):
df, chiSquared = chi_squared(actual, expected) df, chiSquared = chi_squared(actual, expected)
if chiSquared >= _chiSquared_table[df]: if chiSquared >= _chiSquared_table[df]:
print('Significant difference between expected and actual answer distributions: \n' + if show:
'Chi2 value: {} with {} degrees of freedom'.format(chiSquared, df)) print('Significant difference between expected and actual answer distributions: \n' +
'Chi2 value: {} with {} degrees of freedom'.format(chiSquared, df))
return False return False
return True return True
def cross_formula_chi_squared(actualDict, expectedDict): def cross_formula_chi_squared(actualDict, expectedDict):
for ka, actual in actualDict.items(): for ka, actual in actualDict.items():
for ke, expected in expectedDict.items(): for ke, expected in expectedDict.items():
print('Comparing {} with {}'.format(ka, ke)) print('Comparing {} with {}: '.format(ka, ke), end='')
chi_squared_test(actual, expected) if not chi_squared_test(actual, expected, show=False):
print('Failed.')
else:
print('Succeeded.')
def cross_chi_squared(problemSets): def cross_chi_squared(problemSets):
for i, problemSetA in enumerate(problemSets): for i, (a, problemSetA) in enumerate(problemSets):
for problemSetB in problemSets[i + 1:]: for b, problemSetB in problemSets[i + 1:]:
for problemA in problemSetA: for problemA in problemSetA:
for problemB in problemSetB: for problemB in problemSetB:
if (problemA.initial == problemB.initial and if (problemA.initial == problemB.initial and
@ -56,7 +60,14 @@ def cross_chi_squared(problemSets):
problemA.target == problemB.target): problemA.target == problemB.target):
answersA = problemA.distributions answersA = problemA.distributions
answersB = problemB.distributions answersB = problemB.distributions
print('-' * 80)
print('\n')
print('{} x {}'.format(a, b))
print('Problem: {}:{}::{}:_'.format(problemA.initial,
problemA.modified,
problemA.target))
cross_formula_chi_squared(answersA, answersB) cross_formula_chi_squared(answersA, answersB)
print('\n')
def iso_chi_squared(actualDict, expectedDict): def iso_chi_squared(actualDict, expectedDict):
for key in expectedDict.keys(): for key in expectedDict.keys():

2
cross_compare.py
View File

@ -15,7 +15,7 @@ def main(args):
with open(filename, 'rb') as infile: with open(filename, 'rb') as infile:
pSet = pickle.load(infile) pSet = pickle.load(infile)
branchProblemSets[filename] = pSet branchProblemSets[filename] = pSet
problemSets.append(pSet) problemSets.append((filename, pSet))
cross_chi_squared(problemSets) cross_chi_squared(problemSets)
return 0 return 0

BIN
distributions/.adj-tests
View File

Binary file not shown.

292
results.txt Normal file
View File

@ -0,0 +1,292 @@
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.adj-tests
Problem: abc:abd::efg:_
Comparing None with None: Warning! Expected 0 counts of ffg, but got 1
Warning! Expected 0 counts of dfg, but got 2
Warning! Expected 0 counts of efg, but got 1
Warning! Expected 0 counts of eff, but got 6
Succeeded.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.adj-tests
Problem: abc:abd::ijk:_
Comparing None with None: Warning! Expected 0 counts of jjk, but got 4
Warning! Expected 0 counts of ijj, but got 4
Warning! Expected 0 counts of hjk, but got 1
Succeeded.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.adj-tests
Problem: abc:abd::xyz:_
Comparing None with None: Warning! Expected 0 counts of xd, but got 1
Warning! Expected 0 counts of xyz, but got 5
Warning! Expected 0 counts of dyz, but got 1
Warning! Expected 0 counts of yyz, but got 9
Warning! Expected 0 counts of xyy, but got 5
Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.adj-tests
Problem: abc:abd::ijkk:_
Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18
Warning! Expected 0 counts of ijkd, but got 1
Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.adj-tests
Problem: abc:abd::mrrjjj:_
Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 8
Warning! Expected 0 counts of mrrjjd, but got 1
Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.nuke-temp
Problem: abc:abd::efg:_
Comparing None with None: Warning! Expected 0 counts of dfg, but got 2
Warning! Expected 0 counts of efg, but got 1
Warning! Expected 0 counts of eff, but got 6
Succeeded.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.nuke-temp
Problem: abc:abd::ijk:_
Comparing None with None: Warning! Expected 0 counts of jjk, but got 4
Warning! Expected 0 counts of ijj, but got 4
Warning! Expected 0 counts of hjk, but got 1
Succeeded.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.nuke-temp
Problem: abc:abd::xyz:_
Comparing None with None: Warning! Expected 0 counts of xd, but got 1
Warning! Expected 0 counts of xyz, but got 5
Warning! Expected 0 counts of dyz, but got 1
Warning! Expected 0 counts of yyz, but got 9
Warning! Expected 0 counts of xyy, but got 5
Warning! Expected 0 counts of wyz, but got 9
Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.nuke-temp
Problem: abc:abd::ijkk:_
Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18
Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.nuke-temp
Problem: abc:abd::mrrjjj:_
Comparing None with None: Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.soft-remove
Problem: abc:abd::efg:_
Comparing None with None: Warning! Expected 0 counts of ffg, but got 1
Warning! Expected 0 counts of dfg, but got 2
Warning! Expected 0 counts of efg, but got 1
Warning! Expected 0 counts of eff, but got 6
Succeeded.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.soft-remove
Problem: abc:abd::ijk:_
Comparing None with None: Warning! Expected 0 counts of jjk, but got 4
Warning! Expected 0 counts of ijj, but got 4
Warning! Expected 0 counts of hjk, but got 1
Succeeded.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.soft-remove
Problem: abc:abd::xyz:_
Comparing None with None: Warning! Expected 0 counts of xd, but got 1
Warning! Expected 0 counts of xyz, but got 5
Warning! Expected 0 counts of dyz, but got 1
Warning! Expected 0 counts of xyy, but got 5
Warning! Expected 0 counts of wyz, but got 9
Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.soft-remove
Problem: abc:abd::ijkk:_
Comparing None with None: Warning! Expected 0 counts of ijkk, but got 18
Failed.
--------------------------------------------------------------------------------
distributions/.legacy x distributions/.soft-remove
Problem: abc:abd::mrrjjj:_
Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 8
Warning! Expected 0 counts of mrrjjd, but got 1
Failed.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.nuke-temp
Problem: abc:abd::efg:_
Comparing None with None: Succeeded.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.nuke-temp
Problem: abc:abd::ijk:_
Comparing None with None: Succeeded.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.nuke-temp
Problem: abc:abd::xyz:_
Comparing None with None: Warning! Expected 0 counts of wyz, but got 1
Succeeded.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.nuke-temp
Problem: abc:abd::ijkk:_
Comparing None with None: Warning! Expected 0 counts of ijd, but got 1
Warning! Expected 0 counts of ijll, but got 17
Failed.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.nuke-temp
Problem: abc:abd::mrrjjj:_
Comparing None with None: Warning! Expected 0 counts of mrrkkk, but got 26
Failed.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.soft-remove
Problem: abc:abd::efg:_
Comparing None with None: Succeeded.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.soft-remove
Problem: abc:abd::ijk:_
Comparing None with None: Succeeded.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.soft-remove
Problem: abc:abd::xyz:_
Comparing None with None: Warning! Expected 0 counts of wyz, but got 1
Succeeded.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.soft-remove
Problem: abc:abd::ijkk:_
Comparing None with None: Failed.
--------------------------------------------------------------------------------
distributions/.adj-tests x distributions/.soft-remove
Problem: abc:abd::mrrjjj:_
Comparing None with None: Failed.
--------------------------------------------------------------------------------
distributions/.nuke-temp x distributions/.soft-remove
Problem: abc:abd::efg:_
Comparing None with None: Warning! Expected 0 counts of ffg, but got 1
Succeeded.
--------------------------------------------------------------------------------
distributions/.nuke-temp x distributions/.soft-remove
Problem: abc:abd::ijk:_
Comparing None with None: Succeeded.
--------------------------------------------------------------------------------
distributions/.nuke-temp x distributions/.soft-remove
Problem: abc:abd::xyz:_
Comparing None with None: Succeeded.
--------------------------------------------------------------------------------
distributions/.nuke-temp x distributions/.soft-remove
Problem: abc:abd::ijkk:_
Comparing None with None: Failed.
--------------------------------------------------------------------------------
distributions/.nuke-temp x distributions/.soft-remove
Problem: abc:abd::mrrjjj:_
Comparing None with None: Warning! Expected 0 counts of nrrjjj, but got 1
Warning! Expected 0 counts of mrrjjd, but got 2
Failed.