#!/usr/bin/env python3 """ Verify that solutions are stored correctly for different corpora - V1/V2/evaluation: Single grid (one test case) - ConceptArc: Array of grids (multiple test cases) """ import os import json import pymysql from dotenv import load_dotenv def load_env_config(): """Load database configuration from .env file""" load_dotenv() return { 'host': os.getenv('DB_HOST'), 'user': os.getenv('DB_USER'), 'password': os.getenv('DB_PASSWORD'), 'database': os.getenv('DB_NAME'), 'port': int(os.getenv('DB_PORT', 3306)), 'charset': 'utf8mb4' } def check_solution_format(cursor, corpora_name, expected_test_count): """Check solution format for a specific corpora""" print(f"\n{'='*50}") print(f"Checking {corpora_name} puzzles:") print('='*50) cursor.execute(f""" SELECT aj.arc_puzzle_id, aj.json, aj.solution FROM arc_jsons aj JOIN arc_puzzles ap ON aj.arc_puzzle_id = ap.id WHERE ap.corpora = %s LIMIT 5 """, (corpora_name,)) results = cursor.fetchall() if not results: print(f"No {corpora_name} puzzles found") return mismatches = [] for puzzle_id, json_str, solution in results: puzzle_data = json.loads(json_str) test_count = len(puzzle_data.get('test', [])) if not solution: print(f"⚠ {puzzle_id}: No solution stored!") continue sol = json.loads(solution) # Determine solution structure if isinstance(sol, list) and len(sol) > 0: # Check if it's array of grids or single grid if isinstance(sol[0], list) and len(sol[0]) > 0 and isinstance(sol[0][0], list): # Array of grids (ConceptArc style) sol_count = len(sol) structure = f"Array of {sol_count} grids" else: # Single grid (regular ARC style) sol_count = 1 structure = f"Single grid ({len(sol)}x{len(sol[0]) if sol else 0})" else: structure = "Unknown format" sol_count = 0 match = "✓" if sol_count == test_count else "✗" print(f"{match} {puzzle_id}: {test_count} tests, {structure}") if sol_count != test_count: mismatches.append((puzzle_id, test_count, sol_count)) if mismatches: print(f"\n⚠ Found {len(mismatches)} mismatches:") for pid, expected, actual in mismatches: print(f" {pid}: Expected {expected} solutions, got {actual}") else: print(f"\n✓ All solutions match their test counts!") def main(): print("Verifying Solution Formats") print("=" * 50) config = load_env_config() connection = pymysql.connect(**config) cursor = connection.cursor() try: # Check different corpora check_solution_format(cursor, "V1", 1) check_solution_format(cursor, "V2", 1) check_solution_format(cursor, "evaluation", 1) check_solution_format(cursor, "ConceptArc", 3) # Summary stats print(f"\n{'='*50}") print("Summary by corpora:") print('='*50) cursor.execute(""" SELECT ap.corpora, COUNT(*) as total, COUNT(aj.solution) as with_solution FROM arc_puzzles ap JOIN arc_jsons aj ON ap.id = aj.arc_puzzle_id GROUP BY ap.corpora ORDER BY ap.corpora """) for corpora, total, with_sol in cursor.fetchall(): print(f" {corpora}: {with_sol}/{total} have solutions") except Exception as e: print(f"\n✗ Error: {e}") import traceback traceback.print_exc() return 1 finally: connection.close() print(f"\n✓ Database connection closed") return 0 if __name__ == '__main__': import sys sys.exit(main())