Make problem.csv as ground truth and script to test qc accuracy

test_data/AE160707/problems.csv

+record_cxs,problem
+0,missing data

test_data/AE160708/problems.csv

+record_cxs,problem
+#
+3,hatch closed
+4,hatch closed
+5,hatch closed
+6,hatch closed
+7,hatch closed
+8,hatch closed
+1923,hatch closed
+1924,hatch closed
+1925,hatch closed
+1926,hatch closed
+1927,hatch closed
+1928,hatch closed
+1929,hatch closed
+1930,hatch closed
+1933,hatch closed
+1934,hatch closed
+1935,hatch closed
+1936,hatch closed
+1937,hatch closed
+1938,hatch closed
+1939,hatch closed
+1940,hatch closed
+1943,hatch closed
+1944,hatch closed
+1945,hatch closed
+1946,hatch closed
+1947,hatch closed
+1948,hatch closed
+1949,hatch closed
+1950,hatch closed
+1953,hatch closed
+1954,hatch closed
+1955,hatch closed
+1956,hatch closed
+1957,hatch closed
+1958,hatch closed
+1959,hatch closed
+1960,hatch closed
+1963,hatch closed
+1964,hatch closed
+1965,hatch closed
+1966,hatch closed
+1967,hatch closed
+1968,hatch closed
+1969,hatch closed
+1970,hatch closed
+1973,hatch closed
+1974,hatch closed
+1975,hatch closed
+1976,hatch closed
+1977,hatch closed
+1978,hatch closed
+1979,hatch closed
+1980,hatch closed
+1983,hatch closed
+1984,hatch closed
+1985,hatch closed
+1986,hatch closed
+1987,hatch closed
+1988,hatch closed
+1989,hatch closed
+1990,hatch closed
+1993,hatch closed
+1994,hatch closed
+1995,hatch closed
+1996,hatch closed
+1997,hatch closed
+1998,hatch closed
+1999,hatch closed
+2000,hatch closed
+2003,hatch closed
+2004,hatch closed
+2005,hatch closed
+2006,hatch closed
+2007,hatch closed
+2008,hatch closed
+2009,hatch closed
+2010,hatch closed
+2013,hatch closed
+2014,hatch closed
+2015,hatch closed
+2053,hatch closed
+2054,hatch closed
+2055,hatch closed
+2056,hatch closed
+2057,hatch closed
+2058,hatch closed
+2059,hatch closed
+2060,hatch closed
+2063,hatch closed
+2064,hatch closed
+2065,hatch closed
+2066,hatch closed
+2067,hatch closed
+2068,hatch closed
+2069,hatch closed
+2070,hatch closed
+2073,hatch closed
+2074,hatch closed
+2075,hatch closed
+2076,hatch closed
+2077,hatch closed
+2078,hatch closed
+2079,hatch closed
+2080,hatch closed
+2083,hatch closed
+2084,hatch closed
+2085,hatch closed
+2086,hatch closed
+2087,hatch closed
+2088,hatch closed
+2089,hatch closed
+2090,hatch closed
+2093,hatch closed
+2094,hatch closed
+2095,hatch closed
+2096,hatch closed
+2097,hatch closed
+2098,hatch closed
+2099,hatch closed
+2100,hatch closed
+2103,hatch closed
+2104,hatch closed
+2105,hatch closed
+2106,hatch closed
+2107,hatch closed
+2108,hatch closed
+2109,hatch closed
+2110,hatch closed
+2113,hatch closed
+2114,hatch closed
+2115,hatch closed
+2116,hatch closed
+2117,hatch closed
+2118,hatch closed
+2119,hatch closed
+2120,hatch closed
+2123,hatch closed
+2124,hatch closed
+2125,hatch closed
+2126,hatch closed
+2127,hatch closed
+2128,hatch closed
+2129,hatch closed
+2130,hatch closed
+2133,hatch closed
+2134,hatch closed
+2135,hatch closed
+2136,hatch closed
+2137,hatch closed
+2138,hatch closed
+2139,hatch closed
+2140,hatch closed
+2145,hatch closed
+2146,hatch closed
+2147,hatch closed
+2148,hatch closed
+2149,hatch closed
+2150,hatch closed
+2153,hatch closed
+2154,hatch closed
+2155,hatch closed
+2156,hatch closed
+2157,hatch closed
+2158,hatch closed
+2159,hatch closed
+2160,hatch closed
+2163,hatch closed
+2164,hatch closed
+2165,hatch closed
+2166,hatch closed
+2167,hatch closed
+2168,hatch closed
+2169,hatch closed
+2170,hatch closed
+2173,hatch closed
+2174,hatch closed
+2175,hatch closed
+2176,hatch closed
+2177,hatch closed
+2178,hatch closed
+2179,hatch closed
+2180,hatch closed
+2183,hatch closed
+2184,hatch closed
+2185,hatch closed
+2186,hatch closed
+2187,hatch closed
+2188,hatch closed
+2189,hatch closed
+2190,hatch closed
+2193,hatch closed
+2194,hatch closed
+2195,hatch closed
+2196,hatch closed
+2197,hatch closed
+2198,hatch closed
+2199,hatch closed
+2200,hatch closed
+2203,hatch closed
+2204,hatch closed
+2205,hatch closed
+2206,hatch closed
+2207,hatch closed
+2208,hatch closed
+2209,hatch closed
+2210,hatch closed
+2213,hatch closed
+2214,hatch closed
+2215,hatch closed
+2216,hatch closed
+2217,hatch closed
+2218,hatch closed
+2219,hatch closed
+2220,hatch closed
+2223,hatch closed
+2224,hatch closed
+2225,hatch closed
+2226,hatch closed
+2227,hatch closed
+2228,hatch closed
+2229,hatch closed
+2230,hatch closed
+2233,hatch closed
+2234,hatch closed
+2235,hatch closed
+2236,hatch closed
+2237,hatch closed
+2238,hatch closed
+2239,hatch closed
+2240,hatch closed
+2243,hatch closed
+2244,hatch closed
+2245,hatch closed
+2246,hatch closed
+2247,hatch closed
+2248,hatch closed
+2249,hatch closed
+2250,hatch closed
+2253,hatch closed
+2254,hatch closed
+2255,hatch closed
+2256,hatch closed
+2257,hatch closed
+2258,hatch closed
+2259,hatch closed
+2260,hatch closed
+2263,hatch closed
+2264,hatch closed
+2265,hatch closed
+2266,hatch closed
+2267,hatch closed
+2268,hatch closed
+2269,hatch closed
+2270,hatch closed
+2273,hatch closed
+2274,hatch closed
+2275,hatch closed
+2276,hatch closed
+2277,hatch closed
+2278,hatch closed
+2279,hatch closed
+2280,hatch closed
+2283,hatch closed
+2284,hatch closed
+2285,hatch closed
+2286,hatch closed
+2287,hatch closed
+2288,hatch closed
+2289,hatch closed
+2290,hatch closed
+2293,hatch closed
+2294,hatch closed
+2295,hatch closed
+2296,hatch closed
+2297,hatch closed
+2298,hatch closed
+2299,hatch closed
+2300,hatch closed
+2303,hatch closed
+2304,hatch closed
+2305,hatch closed
+2306,hatch closed
+2307,hatch closed
+2308,hatch closed
+2309,hatch closed
+2310,hatch closed
+2313,hatch closed
+2314,hatch closed
+2315,hatch closed
+2316,hatch closed
+2317,hatch closed
+2318,hatch closed
+2319,hatch closed
+2320,hatch closed
+2323,hatch closed
+2324,hatch closed
+2325,hatch closed
+2326,hatch closed
+2327,hatch closed
+2328,hatch closed
+2329,hatch closed
+2330,hatch closed
+2333,hatch closed
+2334,hatch closed
+2335,hatch closed
+2336,hatch closed
+2337,hatch closed
+2338,hatch closed
+2339,hatch closed
+2340,hatch closed
+2343,hatch closed
+2344,hatch closed
+2345,hatch closed
+2346,hatch closed
+2347,hatch closed
+2348,hatch closed
+2349,hatch closed
+2350,hatch closed
+2353,hatch closed
+2354,hatch closed
+2355,hatch closed
+2356,hatch closed
+2357,hatch closed
+2358,hatch closed
+2359,hatch closed
+2360,hatch closed
+2363,hatch closed
+2364,hatch closed
+2365,hatch closed
+2366,hatch closed
+2367,hatch closed
+2368,hatch closed
+2369,hatch closed
+2370,hatch closed
+2373,hatch closed
+2374,hatch closed
+2375,hatch closed
+2376,hatch closed
+2377,hatch closed
+2378,hatch closed
+2379,hatch closed
+2380,hatch closed
+2383,hatch closed
+2384,hatch closed
+2385,hatch closed
+2386,hatch closed
+2387,hatch closed
+2388,hatch closed
+2389,hatch closed
+2390,hatch closed
+2393,hatch closed
+2394,hatch closed
+2395,hatch closed
+2396,hatch closed
+2397,hatch closed
+2398,hatch closed
+2399,hatch closed
+2400,hatch closed
+2403,hatch closed
+2404,hatch closed
+2405,hatch closed
+2406,hatch closed
+2407,hatch closed
+2408,hatch closed
+2409,hatch closed
+2410,hatch closed
+2503,hatch closed
+2504,hatch closed
+2505,hatch closed
+2506,hatch closed
+2507,hatch closed
+2508,hatch closed
+2509,hatch closed
+2510,hatch closed
+2513,hatch closed
+2514,hatch closed
+2515,hatch closed
+2516,hatch closed
+2517,hatch closed
+2518,hatch closed
+2519,hatch closed
+2520,hatch closed
+2523,hatch closed
+2524,hatch closed
+2525,hatch closed
+2526,hatch closed
+2527,hatch closed
+2528,hatch closed
+2529,hatch closed
+2530,hatch closed
+2538,hatch closed
+2539,hatch closed
+2540,hatch closed
+2543,hatch closed
+2544,hatch closed
+2545,hatch closed
+2546,hatch closed
+2547,hatch closed
+2548,hatch closed
+2549,hatch closed
+2550,hatch closed
+2553,hatch closed
+2554,hatch closed
+2555,hatch closed
+2556,hatch closed
+2557,hatch closed
+2558,hatch closed
+2559,hatch closed
+2560,hatch closed
+2563,hatch closed
+2564,hatch closed
+2565,hatch closed
+2566,hatch closed
+2567,hatch closed
+2568,hatch closed
+2569,hatch closed
+2570,hatch closed
+2573,hatch closed
+2574,hatch closed
+2575,hatch closed
+2576,hatch closed
+2577,hatch closed
+2578,hatch closed
+2579,hatch closed
+2580,hatch closed
+2583,hatch closed
+2657,hatch closed
+2658,hatch closed
+2659,hatch closed
+2660,hatch closed
+2663,hatch closed
+2664,hatch closed
+2665,hatch closed
+2666,hatch closed
+2667,hatch closed
+2668,hatch closed
+2669,hatch closed
+2670,hatch closed
+2673,hatch closed
+2674,hatch closed
+2675,hatch closed
+2676,hatch closed
+2677,hatch closed
+2678,hatch closed
+2679,hatch closed
+2680,hatch closed
+2683,hatch closed
+2684,hatch closed
+2685,hatch closed
+2686,hatch closed
+2687,hatch closed
+2688,hatch closed
+2689,hatch closed
+2690,hatch closed
+2693,hatch closed
+2694,hatch closed
+2695,hatch closed
+2696,hatch closed
+2697,hatch closed
+2698,hatch closed
+2699,hatch closed
+2700,hatch closed
+2703,hatch closed
+2704,hatch closed
+2705,hatch closed
+2706,hatch closed
+2707,hatch closed
+2708,hatch closed
+2709,hatch closed
+2710,hatch closed
+2713,hatch closed
+2714,hatch closed
+2715,hatch closed
+2716,hatch closed
+2717,hatch closed
+2719,hatch closed
+2720,hatch closed
+2723,hatch closed
+2724,hatch closed
+2725,hatch closed
+2726,hatch closed
+2727,hatch closed
+2728,hatch closed
+2729,hatch closed
+2730,hatch closed
+2733,hatch closed
+2734,hatch closed
+2735,hatch closed
+2736,hatch closed
+2737,hatch closed
+2738,hatch closed
+2739,hatch closed
+2740,hatch closed
+2743,hatch closed
+2744,hatch closed
+2745,hatch closed
+2746,hatch closed
+2747,hatch closed
+2748,hatch closed
+2749,hatch closed
+2750,hatch closed
+2756,hatch closed
+2757,hatch closed
+2758,hatch closed
+2759,hatch closed
+2762,hatch closed
+2763,hatch closed
+2764,hatch closed
+2765,hatch closed
+2766,hatch closed
+2767,hatch closed
+2768,hatch closed
+2769,hatch closed
+2772,hatch closed
+2773,hatch closed
+2795,hatch closed
+2796,hatch closed
+2797,hatch closed
+2798,hatch closed
+2799,hatch closed
+2802,hatch closed
+2803,hatch closed
+2804,hatch closed
+2805,hatch closed
+2806,hatch closed
+2807,hatch closed
+2808,hatch closed
+2809,hatch closed
+2812,hatch closed
+2813,hatch closed
+2814,hatch closed
+2815,hatch closed
+2816,hatch closed
+2817,hatch closed
+2818,hatch closed
+2819,hatch closed
+2822,hatch closed
+2823,hatch closed
+2824,hatch closed
+2825,hatch closed
+2826,hatch closed
+2827,hatch closed
+2828,hatch closed
+2829,hatch closed
+2837,hatch closed
+2838,hatch closed
+2839,hatch closed
+2842,hatch closed
+2843,hatch closed
+2844,hatch closed
+2845,hatch closed
+2846,hatch closed
+2847,hatch closed
+2848,hatch closed
+2849,hatch closed
+2852,hatch closed
+2853,hatch closed
+2854,hatch closed
+2855,hatch closed
+2856,hatch closed
+2857,hatch closed
+2858,hatch closed
+2859,hatch closed
+3072,hatch closed
+3073,hatch closed
+3074,hatch closed
+3075,hatch closed
+3076,hatch closed
+3077,hatch closed
+3078,hatch closed
+3079,hatch closed
+3082,hatch closed
+3083,hatch closed
+3084,hatch closed
+3085,hatch closed
+3086,hatch closed
+3087,hatch closed
+3088,hatch closed
+3089,hatch closed
+3092,hatch closed
+3093,hatch closed
+3094,hatch closed
+3095,hatch closed
+3096,hatch closed
+3097,hatch closed
+3098,hatch closed
+3099,hatch closed
+3102,hatch closed
+3103,hatch closed
+3104,hatch closed
+3105,hatch closed
+3106,hatch closed
+3107,hatch closed
+3108,hatch closed
+3109,hatch closed
+3112,hatch closed
+3113,hatch closed
+3114,hatch closed
+3143,hatch closed
+3144,hatch closed
+3145,hatch closed
+3146,hatch closed
+3147,hatch closed
+3148,hatch closed
+3149,hatch closed
+3152,hatch closed
+3153,hatch closed
+3154,hatch closed
+3155,hatch closed
+3156,hatch closed
+3157,hatch closed
+3158,hatch closed
+3159,hatch closed
+3162,hatch closed
+3163,hatch closed
+3164,hatch closed
+3165,hatch closed
+3166,hatch closed
+3167,hatch closed
+3168,hatch closed
+3169,hatch closed
+3172,hatch closed
+3173,hatch closed
+3174,hatch closed
+3175,hatch closed
+3176,hatch closed
+3177,hatch closed
+3178,hatch closed
+3179,hatch closed
+3182,hatch closed
+3183,hatch closed
+3184,hatch closed
+3185,hatch closed
+3186,hatch closed
+3187,hatch closed
+3188,hatch closed
+3189,hatch closed
+3192,hatch closed
+3193,hatch closed
+3194,hatch closed
+3195,hatch closed
+3196,hatch closed
+3197,hatch closed
+3198,hatch closed
+3199,hatch closed
+3202,hatch closed
+3203,hatch closed
+3204,hatch closed
+3205,hatch closed
+3206,hatch closed
+3207,hatch closed
+3208,hatch closed
+3209,hatch closed
+3212,hatch closed
+3213,hatch closed
+3214,hatch closed
+3215,hatch closed
+3216,hatch closed
+3217,hatch closed
+3218,hatch closed
+3219,hatch closed
+3222,hatch closed
+3223,hatch closed
+3224,hatch closed
+3225,hatch closed
+3226,hatch closed
+3227,hatch closed
+3228,hatch closed
+3229,hatch closed
+3232,hatch closed
+3233,hatch closed
+3234,hatch closed
+3235,hatch closed
+3236,hatch closed
+3237,hatch closed
+3238,hatch closed
+3239,hatch closed
+3242,hatch closed
+3243,hatch closed
+3244,hatch closed
+3245,hatch closed
+3246,hatch closed
+3247,hatch closed
+3248,hatch closed
+3249,hatch closed
+3252,hatch closed
+3253,hatch closed
+3254,hatch closed
+3255,hatch closed
+3256,hatch closed
+3257,hatch closed
+3258,hatch closed
+3259,hatch closed
+3262,hatch closed
+3263,hatch closed
+3264,hatch closed
+3265,hatch closed
+3266,hatch closed
+3267,hatch closed
+3268,hatch closed
+3269,hatch closed
+3272,hatch closed
+3273,hatch closed
+3274,hatch closed
+3275,hatch closed
+3276,hatch closed
+3277,hatch closed
+3278,hatch closed
+3279,hatch closed
+3282,hatch closed
+3283,hatch closed
+3284,hatch closed
+3285,hatch closed
+3286,hatch closed
+3287,hatch closed
+3288,hatch closed
+3289,hatch closed
+3292,hatch closed
+3293,hatch closed
+3294,hatch closed
+3295,hatch closed
+3296,hatch closed
+3297,hatch closed
+3298,hatch closed
+3299,hatch closed
+3302,hatch closed
+3303,hatch closed
+3304,hatch closed
+3305,hatch closed
+3306,hatch closed
+3307,hatch closed
+3308,hatch closed
+3309,hatch closed
+3312,hatch closed
+3313,hatch closed
+3314,hatch closed
+3315,hatch closed
+3316,hatch closed
+3317,hatch closed
+3318,hatch closed
+3319,hatch closed
+3322,hatch closed
+3323,hatch closed
+3324,hatch closed
+3325,hatch closed
+3326,hatch closed
+3327,hatch closed
+3328,hatch closed
+3329,hatch closed
+3332,hatch closed
+3333,hatch closed
+3334,hatch closed
+3335,hatch closed
+3336,hatch closed
+3337,hatch closed
+3338,hatch closed
+3339,hatch closed
+3342,hatch closed
+3343,hatch closed
+3344,hatch closed
+3345,hatch closed
+3346,hatch closed
+3347,hatch closed
+3348,hatch closed
+3349,hatch closed
+3352,hatch closed
+3353,hatch closed
+3354,hatch closed
+3355,hatch closed
+3356,hatch closed
+3357,hatch closed
+3358,hatch closed
+3359,hatch closed
+3362,hatch closed
+3363,hatch closed
+3364,hatch closed
+3365,hatch closed
+3366,hatch closed
+3367,hatch closed
+3368,hatch closed
+3369,hatch closed
+3372,hatch closed
+3373,hatch closed
+3374,hatch closed
+3375,hatch closed
+3376,hatch closed
+3377,hatch closed
+3378,hatch closed
+3379,hatch closed
+3382,hatch closed
+3383,hatch closed
+3384,hatch closed
+3385,hatch closed
+3386,hatch closed
+3387,hatch closed
+3388,hatch closed
+3389,hatch closed
+3392,hatch closed
+3393,hatch closed
+3394,hatch closed
+3395,hatch closed
+3396,hatch closed
+3397,hatch closed
+3398,hatch closed
+3399,hatch closed
+3402,hatch closed
+3407,hatch closed
+3408,hatch closed
+3409,hatch closed
+3412,hatch closed
+3413,hatch closed
+3414,hatch closed
+3415,hatch closed
+3416,hatch closed
+3417,hatch closed
+3418,hatch closed
+3419,hatch closed
+3422,hatch closed
+3423,hatch closed
+3424,hatch closed
+3425,hatch closed
+3426,hatch closed
+3427,hatch closed
+3428,hatch closed
+3429,hatch closed
+3442,hatch closed
+3443,hatch closed
+3444,hatch closed
+3445,hatch closed
+3446,hatch closed
+3447,hatch closed
+3448,hatch closed
+3449,hatch closed
+3452,hatch closed
+3453,hatch closed
+3454,hatch closed
+3455,hatch closed
+3456,hatch closed
+3457,hatch closed
+3458,hatch closed
+3459,hatch closed
+3897,hatch closed
+3898,hatch closed
+3899,hatch closed
+3902,hatch closed
+3903,hatch closed
+3904,hatch closed
+3905,hatch closed
+3906,hatch closed
+3907,hatch closed
+3908,hatch closed
+3909,hatch closed
+3912,hatch closed
+3913,hatch closed
+3914,hatch closed
+3915,hatch closed
+3916,hatch closed
+3917,hatch closed
+3918,hatch closed
+3919,hatch closed
+3922,hatch closed
+3923,hatch closed
+3924,hatch closed
+3925,hatch closed
+3926,hatch closed
+3927,hatch closed
+3928,hatch closed
+3929,hatch closed
+3932,hatch closed
+3933,hatch closed
+3934,hatch closed
+3935,hatch closed
+3936,hatch closed
+3937,hatch closed
+3938,hatch closed
+3939,hatch closed
+3942,hatch closed
+3962,hatch closed
+3963,hatch closed
+3964,hatch closed
+3965,hatch closed
+3966,hatch closed
+3967,hatch closed
+3968,hatch closed
+3969,hatch closed
+3972,hatch closed
+3973,hatch closed
+3974,hatch closed
+3975,hatch closed
+3976,hatch closed
+3977,hatch closed
+3978,hatch closed
+3979,hatch closed
+#
+1913,safing
+1914,safing
+1915,safing
+1916,safing
+1917,safing
+1918,safing
+1919,safing
+1920,safing
+1921,safing
+1923,safing
+1924,safing
+1925,safing
+1926,safing
+1927,safing
+1928,safing
+1929,safing
+1930,safing
+2043,safing
+2044,safing
+2045,safing
+2046,safing
+2047,safing
+2048,safing
+2049,safing
+2050,safing
+2051,safing
+2053,safing
+2054,safing
+2055,safing
+2056,safing
+2057,safing
+2058,safing
+2059,safing
+2060,safing
+2493,safing
+2494,safing
+2495,safing
+2496,safing
+2497,safing
+2498,safing
+2499,safing
+2500,safing
+2501,safing
+2503,safing
+2504,safing
+2505,safing
+2506,safing
+2507,safing
+2508,safing
+2509,safing
+2510,safing
+3432,safing
+3433,safing
+3434,safing
+3435,safing
+3436,safing
+3437,safing
+3438,safing
+3439,safing
+3440,safing
+3442,safing
+3443,safing
+3444,safing
+3445,safing
+3446,safing
+3447,safing
+3448,safing
+3449,safing
+3952,safing
+3953,safing
+3954,safing
+3955,safing
+3956,safing
+3957,safing
+3958,safing
+3959,safing
+3961,safing
+3962,safing
+3963,safing
+3964,safing
+3965,safing
+3966,safing
+3967,safing
+3968,safing
+3969,safing

test_data/AE160709/problems.csv

+record_cxs,problem
+#
+2,hatch closed
+3,hatch closed
+4,hatch closed
+5,hatch closed
+6,hatch closed
+7,hatch closed
+8,hatch closed
+9,hatch closed
+1559,hatch closed
+1560,hatch closed
+1561,hatch closed
+1562,hatch closed
+1563,hatch closed
+1564,hatch closed
+1565,hatch closed
+1566,hatch closed
+3458,hatch closed
+3459,hatch closed
+3460,hatch closed
+3461,hatch closed
+3462,hatch closed
+3463,hatch closed
+3464,hatch closed
+3465,hatch closed
+3468,hatch closed
+3469,hatch closed
+3470,hatch closed
+#
+3449,safing
+3450,safing
+3451,safing
+3452,safing
+3453,safing
+3454,safing
+3455,safing
+3456,safing
+3457,safing
+3458,safing
+3459,safing
+3460,safing
+3461,safing
+3462,safing
+3463,safing
+3464,safing
+3465,safing
+3466,safing
+3467,safing
+3468,safing
+3469,safing
+3470,safing
+3471,safing
+3472,safing
+3473,safing
+3474,safing
+3475,safing

test_data/AE160710/problems.csv

+record_cxs,problem
+#
+3,hatch closed
+4,hatch closed
+5,hatch closed
+6,hatch closed
+7,hatch closed
+8,hatch closed
+9,hatch closed
+2664,hatch closed
+2665,hatch closed
+2666,hatch closed
+2667,hatch closed
+2668,hatch closed
+2669,hatch closed
+2672,hatch closed
+2673,hatch closed
+2674,hatch closed
+2675,hatch closed
+2676,hatch closed
+2677,hatch closed
+2678,hatch closed

test_data/AE160711/problems.csv

+record_cxs,problem
+#
+2,hatch closed
+3,hatch closed
+4,hatch closed
+5,hatch closed
+6,hatch closed
+7,hatch closed
+4362,hatch closed
+4363,hatch closed
+4364,hatch closed
+4365,hatch closed
+4366,hatch closed
+4367,hatch closed
+4368,hatch closed
+4371,hatch closed
+4372,hatch closed
+4373,hatch closed
+4374,hatch closed
+4375,hatch closed
+4376,hatch closed
+#

test_data/AE160712/problems.csv

+record_cxs,problem
+#
+2,hatch closed
+3,hatch closed
+4,hatch closed
+5,hatch closed
+6,hatch closed
+7,hatch closed
+2553,hatch closed
+2554,hatch closed
+2555,hatch closed
+2556,hatch closed
+2557,hatch closed
+2558,hatch closed
+2561,hatch closed
+2562,hatch closed
+2563,hatch closed
+2564,hatch closed
+2565,hatch closed
+2566,hatch closed
+2567,hatch closed
+3327,hatch closed
+3330,hatch closed
+3331,hatch closed
+3332,hatch closed
+3333,hatch closed
+3334,hatch closed
+3335,hatch closed
+3336,hatch closed
+3337,hatch closed
+4259,hatch closed
+4260,hatch closed
+4261,hatch closed
+4262,hatch closed
+4263,hatch closed
+4264,hatch closed
+4265,hatch closed
+4266,hatch closed
+4269,hatch closed
+4270,hatch closed
+4271,hatch closed
+4272,hatch closed
+#
+3321,safing
+3322,safing
+3323,safing
+3324,safing
+3325,safing
+3326,safing
+3327,safing
+3328,safing
+3330,safing
+3331,safing
+3332,safing
+3333,safing
+3334,safing
+3335,safing
+3336,safing
+3337,safing
+3338,safing
+3339,safing
+3340,safing
+3341,safing
+3342,safing
+3343,safing
+3344,safing
+3345,safing
+3346,safing
+3347,safing
+4250,safing
+4251,safing
+4252,safing
+4253,safing
+4254,safing
+4255,safing
+4256,safing
+4257,safing
+4258,safing
+4259,safing
+4260,safing
+4261,safing
+4262,safing
+4263,safing
+4264,safing
+4265,safing
+4266,safing
+4267,safing
+4268,safing
+4269,safing
+4270,safing
+4271,safing
+4272,safing
+4273,safing
+4274,safing
+4275,safing
+4276,safing

test_data/AE160713/problems.csv

+record_cxs,problem
+#
+2,hatch closed
+3,hatch closed
+4,hatch closed
+5,hatch closed
+6,hatch closed
+7,hatch closed
+1683,hatch closed
+1684,hatch closed
+1685,hatch closed
+1686,hatch closed
+1687,hatch closed
+1688,hatch closed
+1691,hatch closed
+1692,hatch closed
+1693,hatch closed
+1694,hatch closed
+1695,hatch closed
+1696,hatch closed
+1697,hatch closed
+2125,hatch closed
+2126,hatch closed
+2127,hatch closed
+2130,hatch closed
+2131,hatch closed
+2132,hatch closed
+2133,hatch closed
+2134,hatch closed
+2135,hatch closed
+2136,hatch closed
+2137,hatch closed
+2496,hatch closed
+2499,hatch closed
+2500,hatch closed
+2501,hatch closed
+2502,hatch closed
+2503,hatch closed
+2504,hatch closed
+2505,hatch closed
+2506,hatch closed
+2509,hatch closed
+2510,hatch closed
+2699,hatch closed
+2700,hatch closed
+2701,hatch closed
+2702,hatch closed
+2703,hatch closed
+2704,hatch closed
+2705,hatch closed
+3061,hatch closed
+3062,hatch closed
+3063,hatch closed
+3064,hatch closed
+3065,hatch closed
+3066,hatch closed
+3067,hatch closed
+3741,hatch closed
+3742,hatch closed
+3743,hatch closed
+3744,hatch closed
+3745,hatch closed
+3746,hatch closed
+#
+2130,safing
+2131,safing
+2132,safing
+2133,safing
+2134,safing
+2135,safing
+2136,safing
+2137,safing
+2138,safing
+2139,safing
+2140,safing
+2141,safing
+2142,safing
+2143,safing
+2144,safing
+2145,safing
+2146,safing
+2147,safing
+2490,safing
+2491,safing
+2492,safing
+2493,safing
+2494,safing
+2495,safing
+2496,safing
+2497,safing
+2499,safing
+2500,safing
+2501,safing
+2502,safing
+2503,safing
+2504,safing
+2505,safing
+2506,safing
+2507,safing
+2508,safing
+2509,safing
+2510,safing
+2511,safing
+2512,safing
+2513,safing
+2514,safing
+2515,safing
+2516,safing
+#
+3059,HBBbottomTemp outlier
+3060,HBBbottomTemp outlier
+#
+3059,HBBtopTemp outlier
+3060,HBBtopTemp outlier
+#
+3059,HBBapexTemp outlier
+3060,HBBapexTemp outlier

test_data/test_classification.py

+import sklearn.metrics
+import numpy as np
+import pandas as pd
+import netCDF4
+import os
+
+from glob import glob
+
+def main():
+    truths = []
+    qc_percents = []
+    for problem_file in glob('AE*/problems.csv'):
+        for qc_file in glob(os.path.join(os.path.dirname(problem_file), '*.qc')):
+            qc = netCDF4.Dataset(qc_file)
+            break
+        else:
+            continue
+        qc_percent = qc.variables['qc_percent'][:]
+        problem = pd.read_csv(problem_file, comment='#')
+        
+        truth = np.in1d(np.arange(len(qc_percent)), problem.record_cxs)
+
+        report = sklearn.metrics.classification_report(truth, qc_percent > .95, target_names=['Pass','Fail'])
+
+        print('-'*50)
+        print(os.path.dirname(problem_file))
+        print(report)
+        print('-'*50)
+        
+        truths.append(truth)
+        qc_percents.append(qc_percent)
+
+    overall_report = sklearn.metrics.classification_report(np.concatenate(truths), np.concatenate(qc_percents) > .95, target_names=['Pass','Fail'])
+    print('='*50)
+    print(overall_report)
+    print('='*50)
+
+if __name__ == '__main__':
+    main()