tests.data source code

1 // This file is licensed under the Boost License, with code adopted from Silly (https://gitlab.com/AntonMeep/silly),
2 // which is licensed under the ISC license:
3 // Copyright (c) 2019, Anton Fediushin
4 //
5 // Permission to use, copy, modify, and/or distribute this software for any
6 // purpose with or without fee is hereby granted, provided that the above
7 // copyright notice and this permission notice appear in all copies.
8 //
9 // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 
17 module tests.data;
18 import tests.utils;
19 import std.path : buildPath;
20 
21 __gshared string testDataLocation = "../ion-tests/iontestdata";
22 
23 enum IonTestData {
24     good,
25     goodTypecodes,
26     goodTimestamp,
27     bad,
28     badTypecodes,
29     badTimestamp,
30     equivs,
31     nonequivs,
32     roundtrip
33 };
34 
35 // iontestdata/good
36 enum ION_GOOD_TEST_DATA = "good";
37 // iontestdata/good/typecodes
38 enum ION_GOOD_TYPECODES_TEST_DATA = buildPath(ION_GOOD_TEST_DATA, "typecodes");
39 // iontestdata/good/timestamp
40 enum ION_GOOD_TIMESTAMP_TEST_DATA = buildPath(ION_GOOD_TEST_DATA, "timestamp");
41 // iontestdata/bad
42 enum ION_BAD_TEST_DATA = "bad";
43 // iontestdata/bad/typecodes
44 enum ION_BAD_TYPECODES_TEST_DATA = buildPath(ION_BAD_TEST_DATA, "typecodes");
45 // iontestdata/bad/timestamp
46 enum ION_BAD_TIMESTAMP_TEST_DATA = buildPath(ION_BAD_TEST_DATA, "timestamp");
47 // iontestdata/good/equivs
48 enum ION_EQUIVS_TEST_DATA = buildPath(ION_GOOD_TEST_DATA, "equivs");
49 // iontestdata/good/non-equivs
50 enum ION_NONEQUIVS_TEST_DATA = buildPath(ION_GOOD_TEST_DATA, "non-equivs");
51 enum ION_ROUNDTRIP_TEST_DATA = "good";
52 
53 static immutable const(char)[][] ION_TEST_DATA = [
54     ION_GOOD_TEST_DATA,
55     ION_GOOD_TYPECODES_TEST_DATA,
56     ION_GOOD_TIMESTAMP_TEST_DATA,
57     ION_BAD_TEST_DATA,
58     ION_BAD_TYPECODES_TEST_DATA,
59     ION_BAD_TIMESTAMP_TEST_DATA,
60     ION_EQUIVS_TEST_DATA,
61     ION_NONEQUIVS_TEST_DATA,
62     ION_ROUNDTRIP_TEST_DATA
63 ];
64 
65 static immutable ION_GOOD_TEST_DATA_SKIP = [
66     // upstream implementations can't parse these files, don't bother
67     "good/subfieldVarUInt32bit.ion",
68     "good/utf16.ion",
69     "good/utf32.ion",
70     "good/whitespace.ion",
71     "good/item1.10n",
72     "good/testfile26.ion",
73     "good/localSymbolTableImportZeroMaxId.ion",
74     // Shared symbol tables support is TBD
75     "good/subfieldVarUInt.ion",
76     "good/subfieldVarUInt15bit.ion",
77     "good/testfile35.ion",
78     "good/subfieldVarUInt16bit.ion",
79     // We shouldn't have a IonValueStream that's fully empty in real data
80     "good/empty.ion",
81     "good/blank.ion",
82     // Mir supports up to 1024 bytes big integers/decimal for coefficient.
83     // This test requires 1201 bytes for coefficient.
84     "good/intBigSize1201.10n",
85 ];
86 
87 static immutable ION_GOOD_TYPECODES_TEST_DATA_SKIP = [];
88 
89 static immutable ION_GOOD_TIMESTAMP_TEST_DATA_SKIP = [];
90 
91 static immutable ION_BAD_TEST_DATA_SKIP = [
92     "bad/clobWithNullCharacter.ion"
93 ];
94 
95 static immutable ION_BAD_TYPECODES_TEST_DATA_SKIP = [
96     "bad/typecodes/type_6_length_0.10n"
97 ];
98 
99 static immutable ION_BAD_TIMESTAMP_TEST_DATA_SKIP = [];
100 
101 static immutable ION_EQUIVS_TEST_DATA_SKIP = [
102     "good/equivs/clobNewlines.ion",
103 ];
104 
105 static immutable ION_NONEQUIVS_TEST_DATA_SKIP = [];
106 
107 static immutable ION_ROUNDTRIP_TEST_DATA_SKIP = ION_GOOD_TEST_DATA_SKIP ~ [
108     "good/testfile37.ion",
109     "good/testfile23.ion",
110 ];
111 
112 static immutable const(char)[][][] ION_TEST_DATA_SKIP = [
113     ION_GOOD_TEST_DATA_SKIP,
114     ION_GOOD_TYPECODES_TEST_DATA_SKIP,
115     ION_GOOD_TIMESTAMP_TEST_DATA_SKIP,
116     ION_BAD_TEST_DATA_SKIP,
117     ION_BAD_TYPECODES_TEST_DATA_SKIP,
118     ION_BAD_TIMESTAMP_TEST_DATA_SKIP,
119     ION_EQUIVS_TEST_DATA_SKIP,
120     ION_NONEQUIVS_TEST_DATA_SKIP,
121     ION_ROUNDTRIP_TEST_DATA_SKIP,
122 ];
123 
124 bool isSkippedFile(IonTestData testData, string path) {
125     import std.algorithm.searching : any, canFind;
126     return ION_TEST_DATA_SKIP[testData].any!(e => path.canFind(e));
127 }
128 
129 enum IonDataType {
130     binary,
131     text,
132     all
133 };
134 
135 struct Test {
136     string filePath;
137     string name;
138     bool verbose;
139     bool expectedFail;
140     IonDataType type;
141     ubyte[] data;
142 
143     void run(alias m)(out TestResult result, bool failFast, bool verbose) {
144         result.test = this;
145         this.verbose = verbose;
146 
147         try {
148             m(this);
149 
150             if (!expectedFail) {
151                 result.passed = true;
152             }
153         } catch (Throwable t) {
154             if (expectedFail) {
155                 result.passed = true;
156             }
157 
158             import core.exception : AssertError;
159             foreach(th; t) {
160                 Thrown thrown;
161 
162                 foreach(exc; th.info) {
163                     thrown.info ~= exc.idup;
164                 }
165                 thrown.type = typeid(th).name;
166                 thrown.file = th.file;
167                 thrown.message = th.message.idup;
168                 thrown.line = th.line;
169 
170                 result.thrown ~= thrown; 
171             }
172 
173             if (!(cast(Exception) t || cast(AssertError) t)) {
174                 throw t;
175             } else if (failFast && !expectedFail) {
176                 throw t;
177             }
178         }
179     }
180 }
181 
182 struct Thrown {
183 	string type;
184     string message;
185 	string file;
186 	size_t line;
187 	immutable(string)[] info;
188 }
189 
190 struct TestResult {
191     Test test;
192     bool passed;
193     immutable(Thrown)[] thrown;
194 
195     void print(bool failuresOnly = false, bool verbose = false) {
196         import std.format : formattedWrite;
197         import std.stdio : stdout;
198         if (failuresOnly && passed)
199             return;
200 
201         auto writer = stdout.lockingTextWriter;
202         writer.formattedWrite("[%s] %s\n",
203                 passed ? "✓".okayText 
204                        : "✗".failText,
205                 test.name.emphasizeText,
206                 test.filePath);
207 
208         // If this is an expected failure test case AND we want verbose messaging,
209         // or if this is just a plain test case, then print it out
210         if (test.expectedFail && !passed) {
211             writer.formattedWrite("    expectedFail: Test did not throw when it was expected to.\n");
212         }
213         else if ((test.expectedFail && verbose) || (!test.expectedFail)) {
214             foreach(th; thrown) {
215                 writer.formattedWrite("    %s: %s (file: %s:%d)\n", 
216                         th.type,
217                         th.message,
218                         th.file,
219                         th.line);
220 
221                 if (verbose) { 
222                     writer.formattedWrite("    ------ STACK TRACE -----\n");
223                     foreach(line; th.info) {
224                         writer.formattedWrite("    %s\n", line);
225                     }
226                 }
227             }
228         }
229     }
230 }
231 
232 Test[] loadIonTestData(string root, IonTestData dataType, bool expectedFail, IonDataType wantedType) {
233     import std.array : array, join;
234     import std.file : read, dirEntries, SpanMode, DirEntry;
235     import std.path : buildPath, relativePath, extension;
236     string path = buildPath(root, ION_TEST_DATA[dataType]);
237     Test[] testCases;
238     string searchPattern = "*{.ion,.10n}";
239     if (wantedType == IonDataType.binary)
240         searchPattern = "*{.10n}";
241     else if (wantedType == IonDataType.text)
242         searchPattern = "*{.ion}";
243 
244     foreach (DirEntry e; dirEntries(path, searchPattern, SpanMode.shallow)) {
245         if (dataType.isSkippedFile(e.name)) {
246             continue;
247         }
248 
249         Test testCase;
250         testCase.filePath = e.name;
251         testCase.name = relativePath(e.name, testDataLocation);
252         testCase.data = cast(ubyte[])read(e.name);
253         testCase.type = e.name.extension == ".10n" ? IonDataType.binary : IonDataType.text;
254         testCase.expectedFail = expectedFail;
255 
256         testCases ~= testCase;
257     }
258 
259     return testCases;
260 }