1 /++
2 Ion Text Deserialization API
3
4 Heavily influenced (and compatible) with upstream Ion implementations (compatible with ion-go)
5 Authors: Harrison Ford
6 +/
7 module mir.deser.text;
8 import mir.deser.text.readers;
9 import mir.deser.text.skippers;
10 import mir.deser.text.tokenizer;
11 import mir.deser.text.tokens;
12 import mir.ion.symbol_table;
13 import mir.ion.type_code;
14 import mir.ion.tape;
15 import mir.ser.ion;
16 import mir.serde;
17 import mir.format; // Quoted symbol support
18 import mir.ion.internal.data_holder;
19 import mir.ion.internal.stage3 : IonErrorInfo;
20 import mir.bignum.integer;
21 import std.traits : hasUDA, getUDAs;
22 import mir.exception: toMutable;
23
24 private mixin template Stack(T, size_t maxDepth = 1024)
25 {
26 private
27 {
28 immutable maxDepthReachedMsg = "max depth on stack reached";
29 immutable cannotPopNoElementsMsg = "cannot pop from stack with 0 elements";
30 version(D_Exceptions) {
31 immutable maxDepthException = new Exception(maxDepthReachedMsg);
32 immutable cannotPopNoElementsException = new Exception(cannotPopNoElementsMsg);
33 }
34 T[maxDepth] stack;
35 size_t stackLength;
36 }
37
38 T peekStack() @safe @nogc pure
39 {
40 if (stackLength == 0) return T.init;
41 return stack[stackLength - 1];
42 }
43
44 void pushStack(T element) @safe @nogc pure
45 {
46 if (stackLength + 1 > maxDepth) {
47 version(D_Exceptions)
48 throw maxDepthException.toMutable;
49 else
50 assert(0, maxDepthReachedMsg);
51 }
52 stack[stackLength++] = element;
53 }
54
55 T popStackBack() @safe @nogc pure
56 {
57 if (stackLength <= 0) {
58 version (D_Exceptions)
59 throw cannotPopNoElementsException.toMutable;
60 else
61 assert(0, cannotPopNoElementsMsg);
62 }
63 T code = stack[stackLength];
64 stack[--stackLength] = T.init;
65 return code;
66 }
67 }
68
69 private enum State {
70 beforeAnnotations,
71 beforeFieldName,
72 beforeContainer,
73 afterValue,
74 EOF
75 }
76
77 // UDA
78 private struct S
79 {
80 State state;
81 bool transition = false;
82 bool disableAfterValue = false;
83 }
84
85 /++
86 Deserializer for the Ion Text format
87 +/
88 struct IonTextDeserializer(Serializer)
89 {
90 mixin Stack!(IonTypeCode);
91 private Serializer* ser;
92 private State state;
93 private IonTokenizer t;
94
95 /++
96 Constructor
97 Params:
98 ser = A pointer to a serializer
99 +/
100 this(Serializer* ser) @safe pure
101 {
102 this.ser = ser;
103 this.state = State.beforeAnnotations;
104 }
105
106 /++
107 This function starts the deserializing process, and attempts to fully read through
108 the text provided until it reaches the end.
109 Params:
110 text = The text to deserialize
111 +/
112 void opCall(scope const(char)[] text) @trusted pure
113 {
114 t = IonTokenizer(text);
115 while (!t.isEOF())
116 {
117 auto ntr = t.nextToken();
118 assert(ntr, "hit eof when tokenizer says we're not at an EOF??");
119
120 switch (state) with (State)
121 {
122 case afterValue:
123 case beforeFieldName:
124 case beforeAnnotations:
125 handleState(state);
126 break;
127 default:
128 version(D_Exceptions)
129 throw IonDeserializerErrorCode.unexpectedState.ionDeserializerException;
130 else
131 assert(0, "unexpected state");
132 }
133 }
134 }
135
136 private:
137
138 static void __bar()
139 {
140 typeof(*typeof(this).init.ser).init.putAnnotation("symbolText");
141 }
142
143 static assert(__traits(compiles, (){__bar();}));
144
145 // import std.traits:
146 static if (__traits(compiles, ()@nogc{__bar();}))
147 {
148 void handleState(State s) @safe pure @nogc { handleStateImpl(s); }
149 void handleToken(IonTokenType t) @safe pure @nogc { handleTokenImpl(t); }
150 }
151 else
152 {
153 void handleState(State s) @safe pure { handleStateImpl(s); }
154 void handleToken(IonTokenType t) @safe pure { handleTokenImpl(t); }
155 }
156
157 private void handleStateImpl(State s) @safe pure
158 {
159 switch (s) {
160 // Cannot use getSymbolsByUDA as it leads to recursive template instantiations
161 static foreach(member; __traits(allMembers, typeof(this))) {
162 static foreach(registeredState; getUDAs!(__traits(getMember, this, member), S))
163 static if (!registeredState.transition) {
164 case registeredState.state:
165 __traits(getMember, this, member)();
166 static if (!registeredState.disableAfterValue)
167 state = handleStateTransition(State.afterValue);
168 return;
169 }
170 }
171 default: {
172 version (D_Exceptions)
173 throw IonDeserializerErrorCode.unexpectedState.ionDeserializerException;
174 else
175 assert(0, "Unexpected state");
176 }
177 }
178 }
179
180 private State handleStateTransition(State s) @safe pure
181 {
182 switch (s)
183 {
184 static foreach(member; __traits(allMembers, typeof(this)))
185 static foreach(registeredState; getUDAs!(__traits(getMember, this, member), S))
186 static if (registeredState.transition) {
187 case registeredState.state:
188 return __traits(getMember, this, member);
189 }
190 default:
191 version (D_Exceptions)
192 throw IonDeserializerErrorCode.unexpectedState.ionDeserializerException;
193 else
194 assert(0, "Unexpected state");
195 }
196 }
197
198 private void handleTokenImpl(IonTokenType t) @safe pure
199 {
200 switch (t)
201 {
202 static foreach(member; __traits(allMembers, typeof(this)))
203 static foreach(registeredToken; getUDAs!(__traits(getMember, this, member), IonTokenType)) {
204 case registeredToken:
205 return __traits(getMember, this, member);
206 }
207 default:
208 version (D_Exceptions)
209 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
210 else
211 assert(0, "Unexpected token");
212 }
213 }
214
215 /* State / state transition handlers */
216
217 @S(State.beforeAnnotations)
218 bool handleBeforeAnnotations() @safe pure
219 {
220 switch (t.currentToken) with (IonTokenType)
221 {
222 case TokenString:
223 case TokenLongString:
224 case TokenTimestamp:
225 case TokenBinary:
226 case TokenHex:
227 case TokenNumber:
228 case TokenFloatInf:
229 case TokenFloatMinusInf:
230 case TokenFloatNaN:
231 case TokenSymbolQuoted:
232 case TokenSymbol:
233 case TokenSymbolOperator:
234 case TokenDot:
235 case TokenOpenDoubleBrace:
236 case TokenOpenBrace:
237 case TokenOpenBracket:
238 case TokenOpenParen:
239 handleToken(t.currentToken);
240 return true;
241 case TokenEOF:
242 return false;
243 default:
244 version(D_Exceptions)
245 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
246 else
247 assert(0, "unexpected token");
248 }
249 }
250
251 // Typically called within a struct
252 @S(State.beforeFieldName, false, true)
253 bool handleBeforeFieldName() @safe pure
254 {
255 switch (t.currentToken) with (IonTokenType)
256 {
257 case TokenCloseBrace: // Simply just return if this is empty
258 return true;
259 case TokenString:
260 case TokenLongString:
261 {
262 // This code is very similar to the string handling code,
263 // but we put the data on a scoped buffer rather then using the serializer
264 // to put a string by parts.
265 auto buf = stringBuf;
266 IonTextString v;
267 if (t.currentToken == TokenString)
268 {
269 v = t.readValue!(TokenString);
270 }
271 else
272 {
273 v = t.readValue!(TokenLongString);
274 }
275
276 buf.put(v.matchedText);
277 while (!v.isFinal)
278 {
279 if (t.currentToken == IonTokenType.TokenString)
280 {
281 v = t.readValue!(IonTokenType.TokenString);
282 }
283 else
284 {
285 v = t.readValue!(IonTokenType.TokenLongString);
286 }
287 buf.put(v.matchedText);
288 }
289
290 // At this point, we should've fully read out the contents of the first long string,
291 // so we should check if there's any long strings following this one.
292 if (t.currentToken == IonTokenType.TokenLongString)
293 {
294 while (true)
295 {
296 char c = t.skipWhitespace();
297 if (c == '\'')
298 {
299 auto cs = t.peekMax(2);
300 if (cs.length == 2 && cs[0] == '\'' && cs[1] == '\'')
301 {
302 t.skipExactly(2);
303 v = t.readValue!(IonTokenType.TokenLongString);
304 buf.put(v.matchedText);
305 while (!v.isFinal)
306 {
307 v = t.readValue!(IonTokenType.TokenLongString);
308 buf.put(v.matchedText);
309 }
310 }
311 else
312 {
313 t.unread(c);
314 break;
315 }
316 }
317 else
318 {
319 t.unread(c);
320 break;
321 }
322 }
323 }
324
325 ser.putKey(buf.data);
326 if (!t.nextToken())
327 {
328 version(D_Exceptions)
329 throw IonDeserializerErrorCode.unexpectedEOF.ionDeserializerException;
330 else
331 assert(0, "unexpected end of file");
332 }
333 if (t.currentToken != TokenColon)
334 {
335 version(D_Exceptions)
336 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
337 else
338 assert(0, "unexpected token");
339 }
340 state = State.beforeAnnotations;
341 return true;
342 }
343 static foreach(tok; [TokenSymbol, TokenSymbolQuoted])
344 {
345 case tok:
346 {
347 auto val = t.readValue!(tok);
348
349 static if (tok == TokenSymbol)
350 {
351 if (symbolNeedsQuotes(val.matchedText))
352 {
353 version(D_Exceptions)
354 throw IonDeserializerErrorCode.requiresQuotes.ionDeserializerException;
355 else
356 assert(0, "unquoted symbol requires quotes");
357 }
358 ser.putKey(val.matchedText);
359 } else {
360 auto buf = stringBuf;
361 buf.put(val.matchedText);
362 while (!val.isFinal) {
363 val = t.readValue!(tok);
364 buf.put(val.matchedText);
365 }
366 ser.putKey(buf.data);
367 }
368
369 if (!t.nextToken())
370 {
371 version(D_Exceptions)
372 throw IonDeserializerErrorCode.unexpectedEOF.ionDeserializerException;
373 else
374 assert(0, "unexpected end of file");
375 }
376
377 if (t.currentToken != TokenColon)
378 {
379 version(D_Exceptions)
380 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
381 else
382 assert(0, "unexpected token");
383 }
384 state = State.beforeAnnotations;
385 return true;
386 }
387 }
388
389 default:
390 version(D_Exceptions)
391 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
392 else
393 assert(0, "unexpected token");
394 }
395 }
396
397 @S(State.afterValue, false, true)
398 bool handleAfterValue() @safe @nogc pure
399 {
400 switch (t.currentToken) with (IonTokenType)
401 {
402 case TokenComma:
403 auto top = peekStack();
404 if (top == IonTypeCode.struct_)
405 {
406 state = State.beforeFieldName;
407 }
408 else if (top == IonTypeCode.list)
409 {
410 state = State.beforeAnnotations;
411 }
412 else
413 {
414 version(D_Exceptions)
415 throw IonDeserializerErrorCode.unexpectedState.ionDeserializerException;
416 else
417 assert(0, "unexpected state");
418 }
419 return false;
420 case TokenCloseBrace:
421 if (peekStack() == IonTypeCode.struct_)
422 {
423 return true;
424 }
425 goto default;
426 case TokenCloseBracket:
427 if (peekStack() == IonTypeCode.list)
428 {
429 return true;
430 }
431 goto default;
432 case TokenCloseParen:
433 if (peekStack() == IonTypeCode.sexp)
434 {
435 return true;
436 }
437 goto default;
438 default:
439 version(D_Exceptions)
440 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
441 else
442 assert(0, "unexpected token");
443 }
444 }
445
446 @S(State.afterValue, true)
447 State transitionAfterValue() @safe @nogc pure
448 {
449 switch (peekStack()) with (IonTypeCode)
450 {
451 case list:
452 case struct_:
453 return State.afterValue;
454 case sexp:
455 case null_:
456 return State.beforeAnnotations;
457 default:
458 version(D_Exceptions)
459 throw IonDeserializerErrorCode.unexpectedState.ionDeserializerException;
460 else
461 assert(0, "unexpected state");
462 }
463 }
464
465 /* Individual token handlers */
466
467 void onNull() @safe pure
468 {
469 auto cs = t.peekMax(1);
470 // Nulls cannot have any whitespace preceding the dot
471 // This is a workaround, as we skip all whitespace checking for the double-colon
472 if (cs.length == 1 && cs[0] == '.' && !isWhitespace(t.input[t.position - 1 .. t.position][0]))
473 {
474 t.skipOne();
475 if (!t.nextToken())
476 {
477 version(D_Exceptions)
478 throw IonDeserializerErrorCode.unexpectedEOF.ionDeserializerException;
479 else
480 assert(0, "unexpected end of file");
481 }
482
483 if (t.currentToken != IonTokenType.TokenSymbol)
484 {
485 version(D_Exceptions)
486 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
487 else
488 assert(0, "unexpected token");
489 }
490 auto val = t.readValue!(IonTokenType.TokenSymbol);
491 sw: switch (val.matchedText)
492 {
493 static foreach(v; ["null", "bool", "int", "float", "decimal",
494 "timestamp", "symbol", "string", "blob",
495 "clob", "list", "struct", "sexp"])
496 {
497 case v:
498 static if (v == "null" || v == "bool" || v == "float" || v == "struct")
499 {
500 mixin ("ser.putNull(IonTypeCode." ~ v ~ "_);");
501 }
502 else static if (v == "int")
503 {
504 ser.putNull(IonTypeCode.uInt);
505 }
506 else
507 {
508 mixin ("ser.putNull(IonTypeCode." ~ v ~ ");");
509 }
510 break sw;
511 }
512 default:
513 version(D_Exceptions)
514 throw IonDeserializerErrorCode.invalidNullType.ionDeserializerException;
515 else
516 assert(0, "invalid null type specified");
517 }
518 }
519 else
520 {
521 ser.putValue(null);
522 }
523 }
524
525 @(IonTokenType.TokenOpenBrace)
526 void onStruct() @safe pure
527 {
528 auto s0 = ser.structBegin();
529 pushStack(IonTypeCode.struct_);
530 state = State.beforeFieldName;
531 t.finished = true;
532 while (t.nextToken())
533 {
534 if (t.currentToken == IonTokenType.TokenCloseBrace)
535 {
536 t.finished = true;
537 break;
538 }
539
540 handleState(state);
541 }
542 assert(peekStack() == IonTypeCode.struct_, "XXX: should never happen");
543 popStackBack();
544 ser.structEnd(s0);
545 }
546
547 @(IonTokenType.TokenOpenBracket)
548 void onList() @safe pure
549 {
550 auto s0 = ser.listBegin();
551 pushStack(IonTypeCode.list);
552 state = State.beforeAnnotations;
553 t.finished = true;
554 while (t.nextToken())
555 {
556 if (t.currentToken == IonTokenType.TokenCloseBracket)
557 {
558 t.finished = true;
559 break;
560 }
561
562 ser.elemBegin; handleState(state);
563 }
564 assert(peekStack() == IonTypeCode.list, "XXX: should never happen");
565 popStackBack();
566 ser.listEnd(s0);
567 }
568
569 @(IonTokenType.TokenOpenParen)
570 void onSexp() @safe pure
571 {
572 auto s0 = ser.sexpBegin();
573 pushStack(IonTypeCode.sexp);
574 state = State.beforeAnnotations;
575 t.finished = true;
576 while (t.nextToken())
577 {
578 if (t.currentToken == IonTokenType.TokenCloseParen)
579 {
580 t.finished = true;
581 break;
582 }
583
584 ser.sexpElemBegin; handleState(state);
585 }
586 assert(peekStack() == IonTypeCode.sexp, "XXX: should never happen");
587 popStackBack();
588 ser.sexpEnd(s0);
589 }
590
591 @(IonTokenType.TokenSymbolOperator)
592 @(IonTokenType.TokenDot)
593 void onSymbolOperator() @safe pure
594 {
595 if (peekStack() != IonTypeCode.sexp)
596 {
597 version(D_Exceptions)
598 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
599 else
600 assert(0, "unexpected token");
601 }
602 onSymbol();
603 }
604
605 @(IonTokenType.TokenSymbol)
606 @(IonTokenType.TokenSymbolQuoted)
607 void onSymbol() @safe pure
608 {
609 // The use of a scoped buffer is inevitable, as quoted symbols
610 // may contain UTF code points, which we read out separately
611 auto buf = stringBuf;
612 const(char)[] symbolText;
613
614 if (t.currentToken == IonTokenType.TokenSymbol)
615 {
616 IonTextSymbol val = t.readValue!(IonTokenType.TokenSymbol);
617 buf.put(val.matchedText);
618 }
619 else if (t.currentToken == IonTokenType.TokenSymbolOperator || t.currentToken == IonTokenType.TokenDot)
620 {
621 IonTextSymbolOperator val = t.readValue!(IonTokenType.TokenSymbolOperator);
622 buf.put(val.matchedText);
623 }
624 else if (t.currentToken == IonTokenType.TokenSymbolQuoted)
625 {
626 IonTextQuotedSymbol val = t.readValue!(IonTokenType.TokenSymbolQuoted);
627 buf.put(val.matchedText);
628 while (!val.isFinal)
629 {
630 val = t.readValue!(IonTokenType.TokenSymbolQuoted);
631 buf.put(val.matchedText);
632 }
633 }
634 symbolText = buf.data;
635
636 if (t.isDoubleColon())
637 {
638 if (t.currentToken == IonTokenType.TokenSymbol && symbolNeedsQuotes(symbolText))
639 {
640 version(D_Exceptions)
641 throw IonDeserializerErrorCode.requiresQuotes.ionDeserializerException;
642 else
643 assert(0, "unquoted symbol requires quotes");
644 }
645 else if (t.currentToken == IonTokenType.TokenSymbolOperator)
646 {
647 version(D_Exceptions)
648 throw IonDeserializerErrorCode.requiresQuotes.ionDeserializerException;
649 else
650 assert(0, "unquoted symbol requires quotes");
651 }
652 // we are an annotation -- special handling is needed here
653 // since we've identified this symbol to be an annotation,
654 // we technically *aren't* finished with reading out the value
655 // and should not default to skipping over the ending mark
656 // rather, we should skip any whitespace and find the next token
657 // (which is ensured to be a double-colon)
658 if (!t.nextToken())
659 {
660 version(D_Exceptions)
661 throw IonDeserializerErrorCode.unexpectedEOF.ionDeserializerException;
662 else
663 assert(0, "unexpected end of file");
664 }
665
666 if (t.currentToken != IonTokenType.TokenDoubleColon)
667 {
668 version(D_Exceptions)
669 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
670 else
671 assert(0, "unexpected token");
672 }
673
674 size_t wrapperStart = ser.annotationWrapperBegin();
675 ser.putAnnotation(symbolText);
676
677 while (t.nextToken())
678 {
679 // check if the next token read is a candidate for our annotation array
680 if (t.currentToken == IonTokenType.TokenSymbol || t.currentToken == IonTokenType.TokenSymbolQuoted)
681 {
682 buf.reset;
683 if (t.currentToken == IonTokenType.TokenSymbol)
684 {
685 IonTextSymbol val = t.readValue!(IonTokenType.TokenSymbol);
686 buf.put(val.matchedText);
687 }
688 else if (t.currentToken == IonTokenType.TokenSymbolQuoted)
689 {
690 IonTextQuotedSymbol val = t.readValue!(IonTokenType.TokenSymbolQuoted);
691 buf.put(val.matchedText);
692 while (!val.isFinal)
693 {
694 val = t.readValue!(IonTokenType.TokenSymbolQuoted);
695 buf.put(val.matchedText);
696 }
697 }
698
699 // if the symbol we read is followed by a ::, then that means that
700 // this is not the end of our annotation array sequence
701 if (t.isDoubleColon())
702 {
703 // set finished to false so we don't skip over values, rather skip over whitespace
704 if (!t.nextToken())
705 {
706 version(D_Exceptions)
707 throw IonDeserializerErrorCode.unexpectedEOF.ionDeserializerException;
708 else
709 assert(0, "unexpected end of file");
710 }
711
712 if (t.currentToken != IonTokenType.TokenDoubleColon)
713 {
714 version(D_Exceptions)
715 throw IonDeserializerErrorCode.unexpectedToken.ionDeserializerException;
716 else
717 assert(0, "unexpected token");
718 }
719 ser.putAnnotation(buf.data);
720 }
721 else
722 {
723 // if not, this is where we end
724 auto arrayStart = ser.annotationsEnd(wrapperStart);
725 ser.putSymbol(buf.data);
726 ser.annotationWrapperEnd(arrayStart, wrapperStart);
727 break;
728 }
729 }
730 else
731 {
732 // if the current token is a value type (a non-symbol), then we should also end the annotation array
733 auto arrayStart = ser.annotationsEnd(wrapperStart);
734 handleToken(t.currentToken);
735 ser.annotationWrapperEnd(arrayStart, wrapperStart);
736 break;
737 }
738 }
739 }
740 else
741 {
742 if (t.currentToken == IonTokenType.TokenSymbol
743 || t.currentToken == IonTokenType.TokenSymbolOperator
744 || t.currentToken == IonTokenType.TokenDot)
745 {
746 switch (symbolText)
747 {
748 case "null":
749 onNull();
750 break;
751 case "true":
752 ser.putValue(true);
753 break;
754 case "false":
755 ser.putValue(false);
756 break;
757 default:
758 ser.putSymbol(symbolText);
759 break;
760 }
761 }
762 else
763 {
764 ser.putSymbol(symbolText);
765 }
766 }
767 }
768
769 @(IonTokenType.TokenString)
770 @(IonTokenType.TokenLongString)
771 void onString() @safe pure
772 {
773 IonTextString v;
774 if (t.currentToken == IonTokenType.TokenString)
775 {
776 v = t.readValue!(IonTokenType.TokenString);
777 }
778 else
779 {
780 v = t.readValue!(IonTokenType.TokenLongString);
781 }
782 auto s0 = ser.stringBegin;
783 ser.putStringPart(v.matchedText);
784 while (!v.isFinal)
785 {
786 if (t.currentToken == IonTokenType.TokenString)
787 {
788 v = t.readValue!(IonTokenType.TokenString);
789 }
790 else
791 {
792 v = t.readValue!(IonTokenType.TokenLongString);
793 }
794 ser.putStringPart(v.matchedText);
795 }
796
797 // At this point, we should've fully read out the contents of the first long string,
798 // so we should check if there's any long strings following this one.
799 if (t.currentToken == IonTokenType.TokenLongString)
800 {
801 while (true)
802 {
803 char c = t.skipWhitespace();
804 if (c == '\'')
805 {
806 auto cs = t.peekMax(2);
807 if (cs.length == 2 && cs[0] == '\'' && cs[1] == '\'')
808 {
809 t.skipExactly(2);
810 v = t.readValue!(IonTokenType.TokenLongString);
811 ser.putStringPart(v.matchedText);
812 while (!v.isFinal)
813 {
814 v = t.readValue!(IonTokenType.TokenLongString);
815 ser.putStringPart(v.matchedText);
816 }
817 }
818 else
819 {
820 t.unread(c);
821 break;
822 }
823 }
824 else
825 {
826 t.unread(c);
827 break;
828 }
829 }
830 }
831
832 ser.stringEnd(s0);
833 }
834
835 @(IonTokenType.TokenTimestamp)
836 void onTimestamp() @safe pure
837 {
838 import mir.timestamp : Timestamp;
839 auto v = t.readValue!(IonTokenType.TokenTimestamp);
840 ser.putValue(Timestamp(v.matchedText));
841 }
842
843 @(IonTokenType.TokenNumber)
844 void onNumber() @safe pure
845 {
846 import mir.bignum.integer;
847 import mir.bignum.decimal;
848 import mir.parse;
849 auto v = t.readValue!(IonTokenType.TokenNumber);
850
851 Decimal!128 dec = void;
852 DecimalExponentKey exponentKey;
853 // special values are handled within the tokenizer and emit different token types
854 // i.e. nan == IonTokenType.TokenFloatNaN, +inf == IonTokenType.TokenFloatInf, etc
855 enum bool allowSpecialValues = false;
856 // Ion spec allows this
857 enum bool allowDotOnBounds = true;
858 enum bool allowDExponent = true;
859 enum bool allowStartingPlus = true;
860 enum bool allowUnderscores = true;
861 enum bool allowLeadingZeros = false;
862 enum bool allowExponent = true;
863 // shouldn't be empty anyways, tokenizer wouldn't allow it
864 enum bool checkEmpty = false;
865
866 if (!dec.fromStringImpl!(
867 char,
868 allowSpecialValues,
869 allowDotOnBounds,
870 allowDExponent,
871 allowStartingPlus,
872 allowUnderscores,
873 allowLeadingZeros,
874 allowExponent,
875 checkEmpty
876 )(v.matchedText, exponentKey))
877 {
878 goto unexpected_decimal_value;
879 }
880
881 if (exponentKey == DecimalExponentKey.none)
882 {
883 assert(dec.coefficient.coefficients.length != 1 || dec.coefficient.coefficients[0] != 0);
884 dec.coefficient.sign = dec.coefficient.sign && dec.coefficient.coefficients.length != 0;
885 // this is not a FP, so we can discard the exponent
886 ser.putValue(dec.coefficient);
887 }
888 else if (
889 exponentKey == DecimalExponentKey.d
890 || exponentKey == DecimalExponentKey.D
891 || exponentKey == DecimalExponentKey.dot)
892 {
893 ser.putValue(dec);
894 }
895 else
896 { // floats handle infinity / nan / e / E
897 ser.putValue(cast(double)dec);
898 }
899
900 return;
901
902 unexpected_decimal_value:
903 version(D_Exceptions)
904 throw IonDeserializerErrorCode.unexpectedDecimalValue.ionDeserializerException;
905 else
906 assert(0, "unexpected decimal value");
907 }
908
909 @(IonTokenType.TokenBinary)
910 void onBinaryNumber() @safe pure
911 {
912 auto v = t.readValue!(IonTokenType.TokenBinary);
913 auto sign = v[0] == '-';
914 auto val = BigInt!128.fromBinaryString!true(v[2 + sign .. $]); // skip over the negative + 0b
915 val.sign = sign && val.coefficients.length;
916 ser.putValue(val);
917 }
918
919 @(IonTokenType.TokenHex)
920 void onHexNumber() @safe pure
921 {
922 auto v = t.readValue!(IonTokenType.TokenHex);
923 auto sign = v[0] == '-';
924 auto val = BigInt!128.fromHexString!true(v[2 + sign .. $]); // skip over the 0x
925 val.sign = sign && val.coefficients.length;
926 ser.putValue(val);
927 }
928
929 @(IonTokenType.TokenFloatInf)
930 @(IonTokenType.TokenFloatMinusInf)
931 @(IonTokenType.TokenFloatNaN)
932 void onFloatSpecial() @safe pure
933 {
934 if (t.currentToken == IonTokenType.TokenFloatNaN)
935 {
936 ser.putValue(float.nan);
937 }
938 else if (t.currentToken == IonTokenType.TokenFloatMinusInf)
939 {
940 ser.putValue(-float.infinity);
941 }
942 else
943 {
944 ser.putValue(float.infinity);
945 }
946 }
947
948 @(IonTokenType.TokenOpenDoubleBrace)
949 void onLob() @safe pure
950 {
951 import mir.lob;
952 auto buf = stringBuf;
953
954 char c = t.skipLobWhitespace();
955 if (c == '"')
956 {
957 IonTextClob clob = t.readClob();
958 buf.put(clob.matchedText);
959 while (!clob.isFinal) {
960 clob = t.readClob();
961 buf.put(clob.matchedText);
962 }
963 ser.putValue(Clob(buf.data));
964 }
965 else if (c == '\'')
966 {
967 if (!t.isTripleQuote)
968 t.unexpectedChar(c);
969 // XXX: ScopedBuffer is unavoidable for the implicit concatenation of long clob values.
970 // Replace when we're able to put in a clob by parts (similar to strings)
971 IonTextClob clob = t.readClob!true();
972 buf.put(clob.matchedText);
973 while (!clob.isFinal)
974 {
975 clob = t.readClob!true();
976 buf.put(clob.matchedText);
977 }
978 ser.putValue(Clob(buf.data));
979 }
980 else
981 {
982 import mir.appender : scopedBuffer;
983 import mir.base64 : decodeBase64;
984 // This is most likely a "blob", and we need every single
985 // character to be read correctly, so we will unread this byte.
986 t.unread(c);
987 auto decoded = scopedBuffer!ubyte;
988 IonTextBlob blob = t.readBlob();
989 // Since we don't do any whitespace trimming, we need to do that here...
990 foreach(b; blob.matchedText) {
991 if (b.isWhitespace) {
992 continue;
993 }
994
995 buf.put(b);
996 }
997
998 if (buf.data.length % 4 != 0) {
999 version(D_Exceptions)
1000 throw IonDeserializerErrorCode.invalidBase64Length.ionDeserializerException;
1001 else
1002 assert(0, "invalid Base64 length (maybe missing padding?)");
1003 }
1004 decodeBase64(buf.data, decoded);
1005 ser.putValue(Blob(decoded.data));
1006 }
1007 t.finished = true;
1008 }
1009 }
1010
1011 /++
1012 Deserialize an Ion Text value to a D value.
1013 Params:
1014 value = (optional) value to deserialize
1015 text = The text to deserialize
1016 Returns:
1017 The deserialized Ion Text value
1018 +/
1019 T deserializeText(T)(scope const(char)[] text)
1020 {
1021 import mir.deser.ion;
1022 import mir.ion.conv : text2ion;
1023 import mir.ion.value;
1024 import mir.appender : scopedBuffer;
1025
1026 T value;
1027 deserializeText!T(value, text);
1028 return value;
1029 }
1030
1031 ///ditto
1032 void deserializeText(T)(scope ref T value, scope const(char)[] text)
1033 {
1034 import mir.deser.ion;
1035 import mir.ion.conv : text2ion;
1036 import mir.ion.value;
1037 import mir.appender : scopedBuffer;
1038
1039 auto buf = scopedBuffer!ubyte;
1040 text2ion(text, buf);
1041 return deserializeIon!T(value, buf.data);
1042 }
1043
1044 /// Test struct deserialization
1045 @safe pure
1046 version(mir_ion_parser_test) unittest
1047 {
1048 import mir.ion.value;
1049 static struct Book
1050 {
1051 string title;
1052 bool wouldRecommend;
1053 string description;
1054 uint numberOfNovellas;
1055 double price;
1056 float weight;
1057 string[] tags;
1058 }
1059
1060 static immutable textData = `
1061 {
1062 "title": "A Hero of Our Time",
1063 "wouldRecommend": true,
1064 "description": "",
1065 "numberOfNovellas": 5,
1066 "price": 7.99,
1067 "weight": 6.88,
1068 "tags": ["russian", "novel", "19th century"]
1069 }`;
1070
1071 Book book = deserializeText!Book(textData);
1072 assert(book.description.length == 0);
1073 assert(book.numberOfNovellas == 5);
1074 assert(book.price == 7.99);
1075 assert(book.tags.length == 3);
1076 assert(book.tags[0] == "russian");
1077 assert(book.tags[1] == "novel");
1078 assert(book.tags[2] == "19th century");
1079 assert(book.title == "A Hero of Our Time");
1080 assert(book.weight == 6.88f);
1081 assert(book.wouldRecommend);
1082 }
1083
1084 /// Test @nogc struct deserialization
1085 @safe pure @nogc
1086 version(mir_ion_parser_test) unittest
1087 {
1088 import mir.ion.value;
1089 import mir.bignum.decimal;
1090 import mir.small_string;
1091 import mir.small_array;
1092 import mir.conv : to;
1093 static struct Book
1094 {
1095 SmallString!64 title;
1096 bool wouldRecommend;
1097 SmallString!64 description;
1098 uint numberOfNovellas;
1099 Decimal!1 price;
1100 double weight;
1101 SmallArray!(SmallString!(16), 10) tags;
1102 }
1103
1104 static immutable textData = `
1105 {
1106 "title": "A Hero of Our Time",
1107 "wouldRecommend": true,
1108 "description": "",
1109 "numberOfNovellas": 5,
1110 "price": 7.99,
1111 "weight": 6.88,
1112 "tags": ["russian", "novel", "19th century"]
1113 }`;
1114
1115 Book book = deserializeText!Book(textData);
1116 assert(book.description.length == 0);
1117 assert(book.numberOfNovellas == 5);
1118 assert(book.price.to!double == 7.99);
1119 assert(book.tags.length == 3);
1120 assert(book.tags[0] == "russian");
1121 assert(book.tags[1] == "novel");
1122 assert(book.tags[2] == "19th century");
1123 assert(book.title == "A Hero of Our Time");
1124 assert(book.weight == 6.88f);
1125 assert(book.wouldRecommend);
1126 }
1127
1128 /// Test that strings are being de-serialized properly
1129 version(mir_ion_parser_test) unittest
1130 {
1131 import mir.test: should;
1132 import mir.ion.stream;
1133 import mir.ion.conv : text2ion;
1134 import mir.ser.text;
1135 void test(const(char)[] ionData, const(char)[] expected)
1136 {
1137 const(char)[] output = ionData.text2ion.IonValueStream.serializeText;
1138 output.should == expected;
1139 }
1140
1141 test(`"hello"`, `"hello"`);
1142 test(`"hello\x20world"`, `"hello world"`);
1143 test(`"hello\u2248world"`, `"hello≈world"`);
1144 test(`"hello\U0001F44Dworld"`, `"hello👍world"`);
1145 }
1146
1147 /// Test that timestamps are de-serialized properly
1148 version(mir_ion_parser_test) unittest
1149 {
1150 import mir.test;
1151 import mir.ion.stream;
1152 import mir.ion.conv : text2ion;
1153 import mir.ion.value : IonTimestamp;
1154 import std.datetime.date : TimeOfDay;
1155 import mir.timestamp : Timestamp;
1156 void test(const(char)[] ionData, Timestamp expected)
1157 {
1158 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1159 Timestamp t = ionValue.get!(IonTimestamp).get;
1160 t.should == expected;
1161 }
1162 }
1163
1164 void testFail(const(char)[] ionData, Timestamp expected)
1165 {
1166 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1167 Timestamp t = ionValue.get!(IonTimestamp).get;
1168 assert(expected != t);
1169 }
1170 }
1171
1172 test("2001-01T", Timestamp(2001, 1));
1173 test("2001-01-02", Timestamp(2001, 1, 2));
1174 test("2001-01-02T", Timestamp(2001, 1, 2));
1175 test("2001-01-02T03:04", Timestamp(2001, 1, 2, 3, 4));
1176 test("2001-01-02T03:04Z", Timestamp(2001, 1, 2, 3, 4).withOffset(0));
1177 test("2001-01-02T03:04+00:00", Timestamp(2001, 1, 2, 3, 4).withOffset(0));
1178 test("2001-01-02T03:05+00:01", Timestamp(2001, 1, 2, 3, 4).withOffset(1));
1179 test("2001-01-02T05:05+02:01", Timestamp(2001, 1, 2, 3, 4).withOffset(2*60+1));
1180 test("2001-01-02T03:04:05", Timestamp(2001, 1, 2, 3, 4, 5));
1181 test("2001-01-02T03:04:05Z", Timestamp(2001, 1, 2, 3, 4, 5).withOffset(0));
1182 test("2001-01-02T03:04:05+00:00", Timestamp(2001, 1, 2, 3, 4, 5).withOffset(0));
1183 test("2001-01-02T03:05:05+00:01", Timestamp(2001, 1, 2, 3, 4, 5).withOffset(1));
1184 test("2001-01-02T05:05:05+02:01", Timestamp(2001, 1, 2, 3, 4, 5).withOffset(2*60+1));
1185 test("2001-01-02T03:04:05.666", Timestamp(2001, 1, 2, 3, 4, 5, -3, 666));
1186 test("2001-01-02T03:04:05.666Z", Timestamp(2001, 1, 2, 3, 4, 5, -3, 666).withOffset(0));
1187 test("2001-01-02T03:04:05.666666Z", Timestamp(2001, 1, 2, 3, 4, 5, -6, 666_666).withOffset(0));
1188 test("2001-01-02T03:54:05.666+00:50", Timestamp(2001, 1, 2, 3, 4, 5, -3, 666).withOffset(50));
1189 test("2001-01-02T03:54:05.666666+00:50", Timestamp(2001, 1, 2, 3, 4, 5, -6, 666_666).withOffset(50));
1190
1191 // Time of day tests
1192 test("03:04", Timestamp(0, 0, 0, 3, 4));
1193 test("03:04Z", Timestamp(0, 0, 0, 3, 4).withOffset(0));
1194 test("03:04+00:00", Timestamp(0, 0, 0, 3, 4).withOffset(0));
1195 test("03:05+00:01", Timestamp(0, 0, 0, 3, 4).withOffset(1));
1196 test("05:05+02:01", Timestamp(0, 0, 0, 3, 4).withOffset(2*60+1));
1197 test("03:04:05", Timestamp(0, 0, 0, 3, 4, 5));
1198 test("03:04:05Z", Timestamp(0, 0, 0, 3, 4, 5).withOffset(0));
1199 test("03:04:05+00:00", Timestamp(0, 0, 0, 3, 4, 5).withOffset(0));
1200 test("03:05:05+00:01", Timestamp(0, 0, 0, 3, 4, 5).withOffset(1));
1201 test("05:05:05+02:01", Timestamp(0, 0, 0, 3, 4, 5).withOffset(2*60+1));
1202 test("03:04:05.666", Timestamp(0, 0, 0, 3, 4, 5, -3, 666));
1203 test("03:04:05.666Z", Timestamp(0, 0, 0, 3, 4, 5, -3, 666).withOffset(0));
1204 test("03:04:05.666666Z", Timestamp(0, 0, 0, 3, 4, 5, -6, 666_666).withOffset(0));
1205 test("03:54:05.666+00:50", Timestamp(0, 0, 0, 3, 4, 5, -3, 666).withOffset(50));
1206 test("03:54:05.666666+00:50", Timestamp(0, 0, 0, 3, 4, 5, -6, 666_666).withOffset(50));
1207
1208 // Mir doesn't like 03:04 only (as technically it's less precise then TimeOfDay)... ugh
1209 test("03:04:05", Timestamp(TimeOfDay(3, 4, 5)));
1210 test("03:04:05Z", Timestamp(TimeOfDay(3, 4, 5)).withOffset(0));
1211 test("03:04:05+00:00", Timestamp(TimeOfDay(3, 4, 5)).withOffset(0));
1212 test("03:05:05+00:01", Timestamp(TimeOfDay(3, 4, 5)).withOffset(1));
1213 test("05:05:05+02:01", Timestamp(TimeOfDay(3, 4, 5)).withOffset(2*60+1));
1214
1215 testFail("2001-01-02T03:04+00:50", Timestamp(2001, 1, 2, 3, 4));
1216 testFail("2001-01-02T03:04:05+00:50", Timestamp(2001, 1, 2, 3, 4, 5));
1217 testFail("2001-01-02T03:04:05.666Z", Timestamp(2001, 1, 2, 3, 4, 5).withOffset(0));
1218 testFail("2001-01-02T03:54:05.666+00:50", Timestamp(2001, 1, 2, 3, 4, 5));
1219
1220 // Fake timestamps for Duration encoding
1221 import core.time : weeks, days, hours, minutes, seconds, hnsecs;
1222 test("0005-02-88T07:40:04.9876543", Timestamp(5.weeks + 2.days + 7.hours + 40.minutes + 4.seconds + 9876543.hnsecs));
1223 test("0005-02-99T07:40:04.9876543", Timestamp(-5.weeks - 2.days - 7.hours - 40.minutes - 4.seconds - 9876543.hnsecs));
1224 }
1225
1226 /// Test that binary literals are de-serialized properly.
1227 version (mir_ion_parser_test) unittest
1228 {
1229 import mir.ion.value : IonUInt;
1230 import mir.ion.stream;
1231 import mir.ion.conv : text2ion;
1232 void test(const(char)[] ionData, uint val)
1233 {
1234 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1235 auto v = ionValue.get!(IonUInt);
1236 assert(v.get!uint == val);
1237 }
1238 }
1239
1240 test("0b00001", 0b1);
1241 test("0b10101", 0b10101);
1242 test("0b11111", 0b11111);
1243 test("0b111111111111111111111", 0b1111_1111_1111_1111_1111_1);
1244 test("0b1_1111_1111_1111_1111_1111", 0b1_1111_1111_1111_1111_1111);
1245 }
1246
1247 /// Test that signed / unsigned integers are de-serialized properly.
1248 version (mir_ion_parser_test) unittest
1249 {
1250 import mir.ion.value : IonUInt, IonNInt;
1251 import mir.ion.stream;
1252 import mir.ion.conv : text2ion;
1253 void test(const(char)[] ionData, ulong val)
1254 {
1255 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1256 auto v = ionValue.get!(IonUInt);
1257 assert(v.get!ulong == val);
1258 }
1259 }
1260
1261 void testNeg(const(char)[] ionData, ulong val)
1262 {
1263 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1264 auto v = ionValue.get!(IonNInt);
1265 assert(v.get!long == -val);
1266 }
1267 }
1268
1269 test("0xabc_def", 0xabc_def);
1270 test("0xabcdef", 0xabcdef);
1271 test("0xDEADBEEF", 0xDEADBEEF);
1272 test("0xDEADBEEF", 0xDEAD_BEEF);
1273 test("0xDEAD_BEEF", 0xDEAD_BEEF);
1274 test("0xDEAD_BEEF", 0xDEADBEEF);
1275 test("0x0123456789", 0x0123456789);
1276 test("0x0123456789abcdef", 0x0123456789abcdef);
1277 test("0x0123_4567_89ab_cdef", 0x0123_4567_89ab_cdef);
1278
1279 testNeg("-0xabc_def", 0xabc_def);
1280 testNeg("-0xabc_def", 0xabc_def);
1281 testNeg("-0xabcdef", 0xabcdef);
1282 testNeg("-0xDEADBEEF", 0xDEADBEEF);
1283 testNeg("-0xDEADBEEF", 0xDEAD_BEEF);
1284 testNeg("-0xDEAD_BEEF", 0xDEAD_BEEF);
1285 testNeg("-0xDEAD_BEEF", 0xDEADBEEF);
1286 testNeg("-0x0123456789", 0x0123456789);
1287 testNeg("-0x0123456789abcdef", 0x0123456789abcdef);
1288 testNeg("-0x0123_4567_89ab_cdef", 0x0123_4567_89ab_cdef);
1289 }
1290
1291 /// Test that infinity & negative infinity are deserialized properly.
1292 version (mir_ion_parser_test) unittest
1293 {
1294 import mir.test: should;
1295 import mir.ion.value : IonFloat;
1296 import mir.ion.conv : text2ion;
1297 import mir.ion.stream;
1298 void test(const(char)[] ionData, float expected)
1299 {
1300 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1301 auto v = ionValue.get!(IonFloat);
1302 v.get!float.should == expected;
1303 }
1304 }
1305
1306 test("-inf", -float.infinity);
1307 test("+inf", float.infinity);
1308 }
1309
1310 /// Test that NaN is deserialized properly.
1311 version (mir_ion_parser_test) unittest
1312 {
1313 import mir.ion.value;
1314 import mir.ion.conv : text2ion;
1315 import mir.ion.stream;
1316
1317 alias isNaN = x => x != x;
1318 void test(const(char)[] ionData)
1319 {
1320 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1321 auto v = ionValue.get!(IonFloat);
1322 assert(isNaN(v.get!float));
1323 }
1324 }
1325
1326 test("nan");
1327 }
1328
1329 /// Test that signed / unsigned integers and decimals and floats are all deserialized properly.
1330 version (mir_ion_parser_test) unittest
1331 {
1332 import mir.test: should;
1333 import mir.ion.value;
1334 import mir.ion.stream;
1335 import mir.ion.conv : text2ion;
1336 void test_uint(const(char)[] ionData, ulong expected)
1337 {
1338 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1339 auto v = ionValue.get!(IonUInt);
1340 v.get!ulong.should == expected;
1341 }
1342 }
1343
1344 void test_nint(const(char)[] ionData, long expected)
1345 {
1346 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1347 auto v = ionValue.get!(IonNInt);
1348 v.get!long.should == expected;
1349 }
1350 }
1351
1352 void test_dec(const(char)[] ionData, double expected)
1353 {
1354 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1355 auto v = ionValue.get!(IonDecimal);
1356 v.get!double.should == expected;
1357 }
1358 }
1359
1360 void test_float(const(char)[] ionData, float expected)
1361 {
1362 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1363 auto v = ionValue.get!(IonFloat);
1364 v.get!float.should == expected;
1365 }
1366 }
1367
1368 test_uint("123", 123);
1369 test_nint("-123", -123);
1370 test_dec("123.123123", 123.123123);
1371 test_dec("123.123123", 123.123123);
1372 test_dec("123.123123d0", 123.123123);
1373 test_dec("123.123123d0", 123.123123);
1374 test_dec("-123.123123", -123.123123);
1375 test_dec("-123.123123d0", -123.123123);
1376 test_dec("18446744073709551615.", 1844_6744_0737_0955_1615.0);
1377 test_dec("-18446744073709551615.", -1844_6744_0737_0955_1615.0);
1378 test_dec("18446744073709551616.", 1844_6744_0737_0955_1616.0);
1379 test_dec("-18446744073709551616.", -1844_6744_0737_0955_1616.0);
1380 test_float("123.456789e-6", 123.456789e-6);
1381 test_float("-123.456789e-6", -123.456789e-6);
1382 }
1383
1384 /// Test that quoted / unquoted symbols are deserialized properly.
1385 version (mir_ion_parser_test) unittest
1386 {
1387 import mir.ion.value;
1388 import mir.ion.conv : text2ion;
1389 import mir.ion.stream;
1390 void test(const(char)[] ionData, string symbol)
1391 {
1392 foreach (symbolTable, val; ionData.text2ion.IonValueStream) {
1393 auto sym = val.get!(IonSymbolID).get;
1394 assert(symbol == symbolTable[sym]);
1395 }
1396 }
1397
1398 test("$0", "$0");
1399 test("$ion", "$ion");
1400 test("$ion_1_0", "$ion_1_0");
1401 test("name", "name");
1402 test("version", "version");
1403 test("imports", "imports");
1404 test("symbols", "symbols");
1405 test("max_id", "max_id");
1406 test("$ion_shared_symbol_table", "$ion_shared_symbol_table");
1407 test("hello", "hello");
1408 test("world", "world");
1409 test("'foobaz'", "foobaz");
1410 test("'👍'", "👍");
1411 test("' '", " ");
1412 test("'\\U0001F44D'", "👍");
1413 test("'\\u2248'", "\u2248");
1414 test("'true'", "true");
1415 test("'false'", "false");
1416 test("'nan'", "nan");
1417 test("'null'", "null");
1418 }
1419
1420 /// Test that all variations of the "null" value are deserialized properly.
1421 version (mir_ion_parser_test) unittest
1422 {
1423 import mir.ion.value;
1424 import mir.ion.stream;
1425 import mir.ion.conv : text2ion;
1426 void test(const(char)[] ionData, IonTypeCode nullType)
1427 {
1428 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1429 auto v = ionValue.get!(IonNull);
1430 assert(v.code == nullType);
1431 }
1432 }
1433
1434 test("null", IonTypeCode.null_);
1435 test("null.bool", IonTypeCode.bool_);
1436 test("null.int", IonTypeCode.uInt);
1437 test("null.float", IonTypeCode.float_);
1438 test("null.decimal", IonTypeCode.decimal);
1439 test("null.timestamp", IonTypeCode.timestamp);
1440 test("null.symbol", IonTypeCode.symbol);
1441 test("null.string", IonTypeCode..string);
1442 test("null.blob", IonTypeCode.blob);
1443 test("null.clob", IonTypeCode.clob);
1444 test("null.list", IonTypeCode.list);
1445 test("null.struct", IonTypeCode.struct_);
1446 test("null.sexp", IonTypeCode.sexp);
1447 }
1448
1449 /// Test that blobs are getting de-serialized correctly.
1450 version (mir_ion_parser_test) unittest
1451 {
1452 import mir.ion.value;
1453 import mir.ion.stream;
1454 import mir.ion.conv : text2ion;
1455 import mir.lob;
1456 void test(const(char)[] ionData, ubyte[] blobData)
1457 {
1458 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1459 auto v = ionValue.get!(Blob);
1460 assert(v.data == blobData);
1461 }
1462 }
1463
1464 test("{{ SGVsbG8sIHdvcmxkIQ== }}", cast(ubyte[])"Hello, world!");
1465 test("{{ R29vZCBhZnRlcm5vb24hIPCfkY0= }}", cast(ubyte[])"Good afternoon! 👍");
1466 }
1467
1468 /// Test that long/short clobs are getting de-serialized correctly.
1469 version (mir_ion_parser_test) unittest
1470 {
1471 import mir.ion.value;
1472 import mir.ion.stream;
1473 import mir.ion.conv : text2ion;
1474 import mir.lob;
1475 void test(const(char)[] ionData, const(char)[] blobData)
1476 {
1477 foreach(symbolTable, scope ionValue; ionData.text2ion.IonValueStream) {
1478 auto v = ionValue.get!(Clob);
1479 assert(v.data == blobData);
1480 }
1481 }
1482
1483 test(`{{ "This is a short clob." }}`, "This is a short clob.");
1484 test(`
1485 {{
1486 '''This is a long clob,'''
1487 ''' which spans over multiple lines,'''
1488 ''' and can have a theoretically infinite length.'''
1489 }}`, "This is a long clob, which spans over multiple lines, and can have a theoretically infinite length.");
1490 test(`{{
1491 '''Long clobs can also have their data contained in one value,
1492 but spread out across multiple lines.'''
1493 }}`, "Long clobs can also have their data contained in one value,\n but spread out across multiple lines.");
1494 test(`{{ '''Or, you can have multiple values on the same line,''' ''' like this!'''}}`,
1495 "Or, you can have multiple values on the same line, like this!");
1496 }
1497
1498 /// Test that structs are getting de-serialized properly
1499 version (mir_ion_parser_test)
1500 unittest
1501 {
1502 import mir.test: should;
1503 import mir.ion.stream;
1504 import mir.ion.conv : text2ion;
1505 import mir.ser.text;
1506 void test(const(char)[] ionData, const(char)[] expected)
1507 {
1508 auto v = ionData.text2ion.IonValueStream.serializeText;
1509 v.should == expected;
1510 }
1511
1512 test(`1`, `1`);
1513 test(`test::1`, `test::1`);
1514
1515 test(`{"test":"world", test: false, 'test': usd::123.456, '''test''': "asdf"}`,
1516 `{test:"world",test:false,test:usd::123.456,test:"asdf"}`);
1517
1518 test(`{'''foo'''
1519 '''bar''': "foobar"}`,
1520 `{foobar:"foobar"}`);
1521
1522 test(`{a: 1, b: 2}`, `{a:1,b:2}`);
1523
1524 test(`{}`, `{}`);
1525 }
1526
1527 /// Test that sexps are getting de-serialized properly.
1528 version (mir_ion_parser_test) unittest
1529 {
1530 import mir.test: should;
1531 import mir.ion.stream;
1532 import mir.ion.conv : text2ion;
1533 import mir.ser.text;
1534 void test(const(char)[] ionData, const(char)[] expected)
1535 {
1536 auto v = ionData.text2ion.IonValueStream.serializeText;
1537 v.should == expected;
1538 }
1539
1540 test(`(this is a sexp list)`, "(this is a sexp list)");
1541 test(`('+' '++' '+-+' '-++' '-' '--' '---' -3 - 3 '--' 3 '--'3 )`,
1542 "('+' '++' '+-+' '-++' '-' '--' '---' -3 '-' 3 '--' 3 '--' 3)");
1543 test(`(a_plus_plus_plus_operator::+++ a_3::3)`, `(a_plus_plus_plus_operator::'+++' a_3::3)`);
1544 test(`(& (% -[42, 3]+(2)-))`, `('&' ('%' '-' [42,3] '+' (2) '-'))`);
1545 }
1546
1547 /// Test that arrays are getting de-serialized properly.
1548 version (mir_ion_parser_test) unittest
1549 {
1550 import mir.test: should;
1551 import mir.ion.stream;
1552 import mir.ion.conv : text2ion;
1553 import mir.ser.text;
1554 void test(const(char)[] ionData, const(char)[] expected)
1555 {
1556 auto v = ionData.text2ion.IonValueStream.serializeText;
1557 v.should == expected;
1558 }
1559
1560 test(`[hello, world]`, `[hello,world]`);
1561 test(`[this::is::an::annotated::symbol, this::is::annotated::123.456]`,
1562 `[this::is::an::annotated::symbol,this::is::annotated::123.456]`);
1563 test(`[date::of::birth::0001-01-01T00:00:00.0+00:00, date::of::birth::1970-01-01T]`,
1564 `[date::of::birth::0001-01-01T00:00:00.0Z,date::of::birth::1970-01-01]`);
1565 test(`['hello', "hello", '''hello''', '''hello ''''''world''']`,
1566 `[hello,"hello","hello","hello world"]`);
1567 test(`[0x123_456, 0xF00D_BAD]`, `[1193046,251714477]`);
1568 }
1569
1570 /// Test that annotations work with symbols
1571 version (mir_ion_parser_test) unittest
1572 {
1573 import mir.test: should;
1574 import mir.ion.stream;
1575 import mir.ser.text;
1576 import mir.ion.conv : text2ion;
1577 void test(const(char)[] ionData, const(char)[] expected)
1578 {
1579 auto v = ionData.text2ion.IonValueStream.serializeText;
1580 v.should == expected;
1581 }
1582
1583 test(`'test'::'hello'::'world'`, "test::hello::world");
1584 test(`foo::bar`, "foo::bar");
1585 test(`foo::'bar'`, "foo::bar");
1586 test(`'foo'::bar`, "foo::bar");
1587 test(`'foo bar'::cash`, "'foo bar'::cash");
1588 test(`'foo\U0001F44D'::'baz\U0001F44D'`, "'foo\U0001F44D'::'baz\U0001F44D'");
1589 test(`'\u2248'::'\u2248'`, "'\u2248'::'\u2248'");
1590 test(`'\u2248'::foo`, "'\u2248'::foo");
1591 }
1592
1593 /// Test that annotations work with floats
1594 version (mir_ion_parser_test) unittest
1595 {
1596 import mir.test: should;
1597 import mir.ion.stream;
1598 import mir.ion.conv : text2ion;
1599 import mir.ser.text;
1600 void test(const(char)[] ionData, const(char)[] expected)
1601 {
1602 auto v = ionData.text2ion.IonValueStream.serializeText;
1603 v.should == expected;
1604 }
1605
1606 test(`usd::10.50e0`, "usd::10.5");
1607 test(`'Value is good \U0001F44D'::12.34e0`, "'Value is good \U0001F44D'::12.34");
1608 test(`'null'::150.00e0`, "'null'::150.0");
1609 }
1610
1611 /// Test that annotations work with decimals
1612 version (mir_ion_parser_test) unittest
1613 {
1614 import mir.test: should;
1615 import mir.ion.stream;
1616 import mir.ion.conv : text2ion;
1617 import mir.ser.text;
1618 void test(const(char)[] ionData, const(char)[] expected)
1619 {
1620 auto v = ionData.text2ion.IonValueStream.serializeText;
1621 v.should == expected;
1622 }
1623
1624 test(`Types::Speed::MetersPerSecondSquared::9.81`, "Types::Speed::MetersPerSecondSquared::9.81");
1625 test(`Rate::USD::GBP::12.345`, "Rate::USD::GBP::12.345");
1626 test(`usd::10.50d0`, "usd::10.50");
1627 test(`'Value is good \U0001F44D'::12.34d0`, "'Value is good \U0001F44D'::12.34");
1628 test(`'null'::150.00d0`, "'null'::150.00");
1629 test(`'Cool'::27182818284590450000000000d-25`, "Cool::2.7182818284590450000000000");
1630 test(`mass::2.718281828459045d0`, "mass::2.718281828459045");
1631 test(`weight::0.000000027182818284590450000000000d+8`, "weight::2.7182818284590450000000000");
1632 test(`coeff::-0.000000027182818284590450000000000d+8`, "coeff::-2.7182818284590450000000000");
1633 }
1634
1635 /// Test that annotations work with strings
1636 version (mir_ion_parser_test) unittest
1637 {
1638 import mir.test: should;
1639 import mir.ion.stream;
1640 import mir.ion.conv : text2ion;
1641 import mir.ser.text;
1642 void test(const(char)[] ionData, const(char)[] expected)
1643 {
1644 auto v = ionData.text2ion.IonValueStream.serializeText;
1645 v.should == expected;
1646 }
1647
1648 test(`Password::"Super Secure Password"`, `Password::"Super Secure Password"`);
1649 test(`Magic::String::"Hello, world!"`, `Magic::String::"Hello, world!"`);
1650 test(`SSH::PublicKey::'''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNrMk7QmmmNIusf10CwHQHs6Z9HJIiuknwoqtQLzEPxdMnNHKJexNnfF5QQ2v84BBhVjxvPgSqhdcVMEFy8PrGu44MqhK/cV6BGx430v2FnArWDO+9LUSd+3iwMJVZUQgZGtjSLAkZO+NOSPWZ+W0SODGgUfbNVu35GjVoA2+e1lOINUe22oZPnaD+gpJGUOx7j5JqpCblBZntvZyOjTPl3pc52rIGfxi1TYJnDXjqX76OinZceBzp5Oh0oUTrPbu55ig+b8bd4HtzLWxcqXBCnsw0OAKsAiXfLlBcrgZUsoAP9unrcqsqoJ2qEEumdsPqcpJakpO7/n0lMP6lRdSZ'''`,
1651 `SSH::PublicKey::"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNrMk7QmmmNIusf10CwHQHs6Z9HJIiuknwoqtQLzEPxdMnNHKJexNnfF5QQ2v84BBhVjxvPgSqhdcVMEFy8PrGu44MqhK/cV6BGx430v2FnArWDO+9LUSd+3iwMJVZUQgZGtjSLAkZO+NOSPWZ+W0SODGgUfbNVu35GjVoA2+e1lOINUe22oZPnaD+gpJGUOx7j5JqpCblBZntvZyOjTPl3pc52rIGfxi1TYJnDXjqX76OinZceBzp5Oh0oUTrPbu55ig+b8bd4HtzLWxcqXBCnsw0OAKsAiXfLlBcrgZUsoAP9unrcqsqoJ2qEEumdsPqcpJakpO7/n0lMP6lRdSZ"`);
1652 }