add some data processing scripts

Author: bitRAKE

scripts/freqdump.g

@@ -0,0 +1,70 @@
+; output byte frequency of input file
+;	fasmg freqdump.g -i "FileName equ 'freqdump.g'"
+;
+; performance:	bytes		seconds
+;		1,126,964	1.2
+;		20,968,922	45.4
+;		510,018,886	534.9
+
+
+match =FileName,FileName
+	display 10,'usage: fasmg freqdump.g -i "FileName equ ''your_file''"',10
+	err 'missing FileName expected'
+else
+	virtual at 0
+		source::
+		file FileName
+	end virtual
+
+	AreaFrequency source
+
+	; we can verify total equals length
+	total = 0
+	postpone
+		assert total = sizeof source
+	end postpone
+
+	; display array of values
+	repeat 256, i:0
+		repeat 1, N:source.i
+			display `N,','
+			total = total + N
+dd N ; and store to output
+		end repeat
+	end repeat
+end match
+
+calminstruction AreaFrequency area*
+	local offset, var, val, i
+	compute offset, 0
+
+	; set all byte frequency counts to zero
+	compute i, 256
+clear:
+	compute i, i - 1
+	arrange var, area.i
+	publish var, offset
+	check i
+	jyes clear
+
+	; increment frequency counts for each byte
+	jump zero
+more:
+	load val, area:offset, 1
+	compute val, 0+val ; convert string to number
+	arrange var, area.val
+	compute val, var + 1
+	publish var, val
+	compute offset, offset + 1
+zero:
+	check offset < sizeof area
+	jyes more
+end calminstruction
+
+;	text AreaContent Some_Area
+;	display text
+calminstruction (target) AreaContent area*
+	local string
+	load string, area: 0, sizeof area
+	publish target, string
+end calminstruction

scripts/json.format.g

@@ -0,0 +1,87 @@
+; strip whitespace from JSON file and re-format with tab
+
+if definite FILE
+virtual
+DAT::
+file FILE
+DAT_END := $ - $$
+end virtual
+
+define _db
+calminstruction json_prettier i:0,instring:0,depth:0
+	local awk,char
+more:
+	arrange awk, =load =_db:1 =from =DAT:i
+	assemble awk
+	compute i, i + 1
+	compute char, 0+_db
+
+	check instring
+	jyes string_contents
+
+	check _db = '"'
+	jyes string_inout
+	check _db = ','
+	jyes output_indent
+
+	check _db = '{'
+	jyes increase_depth
+	check _db = '}'
+	jyes decrease_depth
+
+	check _db = '['
+	jyes increase_depth
+	check _db = ']'
+	jyes decrease_depth
+
+	; consume existing whitespace
+	check _db = ' '
+	jyes skip_ws
+	check _db = 10
+	jyes skip_ws
+	check _db = 13
+	jyes skip_ws
+	check _db = 9
+	jyes skip_ws
+
+output_char:
+	arrange awk, =db char
+	assemble awk
+skip_ws:
+	check i < DAT_END
+	jyes more
+	exit
+
+decrease_depth:
+	compute depth, depth - 1
+indent_output:
+	arrange awk, =db 10,depth =dup 9,char
+	assemble awk
+	jump skip_ws
+
+increase_depth:
+	compute depth, depth + 1
+output_indent:
+	arrange awk, =db char,10,depth =dup 9
+	assemble awk
+	jump skip_ws
+
+string_contents:
+	check instring = 3
+	jyes string_esc
+	check _db = '"'
+	jyes string_inout
+	check _db = '\'
+	jno output_char
+string_esc:
+	compute instring, instring xor 2
+	jump output_char
+string_inout:
+	compute instring, instring xor 1
+	jump output_char
+end calminstruction
+json_prettier
+
+else
+display 'usage: fasmg -i "define FILE ''<filename>''" ',__FILE__,' <outfile>'
+end if

scripts/json.pack.g

@@ -0,0 +1,56 @@
+match =FileName,FileName
+	display 10,'usage: fasmg json.pack.g -i "FileName equ ''metaModel.json''"',10
+	err 'missing FileName expected'
+else
+	virtual at 0
+		source::
+		file FileName
+	end virtual
+end match
+
+calminstruction JSON_Pack area*
+	local offset, outset, val
+	compute outset, -1
+	compute offset, -1
+value:
+	compute offset, offset + 1
+	check offset = sizeof area
+	jyes done
+	load val, area:offset, 1
+	check val = ' '
+	jyes value
+	check val = 9
+	jyes value
+	check val = 10
+	jyes value
+	check val = 13
+	jyes value
+	; store verbatim
+	compute outset, outset + 1
+	store area:outset, 1, val
+	check val = '"'
+	jno value
+quoted: ; assume valid JSON - all quotes
+	compute offset, offset + 1
+	load val, area:offset, 1
+	; store verbatim
+	compute outset, outset + 1
+	store area:outset, 1, val
+	check val = '"'
+	jno quoted
+	; was it an escaped quote?
+	load val, area:offset-1, 1
+	check val = '\'
+	jyes quoted
+	jump value
+done:
+	; define new length to area, SIZEOF no good!
+	compute outset, outset + 1
+	arrange val, area.=packed =:== outset
+	assemble val
+end calminstruction
+
+format binary as 'json'
+JSON_Pack source
+load package:source.packed from source:0
+db package

scripts/makedb.g

@@ -0,0 +1,51 @@
+
+; Make data statements from file (don't use this).
+
+if definite FILE
+virtual at 0
+        DAT::
+        file FILE
+        BYTES := $
+end virtual
+
+db '; ',FILE,13,10
+repeat 1,D:BYTES
+        db 'label dat:',`D,13,10
+end repeat
+db 'db \',13,10
+data_dump BYTES,,db
+
+calminstruction hex_nibble digit*, command: display
+        compute digit, 0FFh and '0123456789ABCDEF' shr (digit*8)
+        arrange command, command digit
+        assemble command
+end calminstruction
+
+define _db
+calminstruction data_dump bytes*, i:0, cmd:display
+        local awk, digit
+jj:     arrange         awk, =load =_db:1 =from =DAT:i
+        assemble        awk
+        arrange         awk, cmd '0x'
+        assemble        awk
+        compute         digit, (_db shr 4) and 0Fh
+        call            hex_nibble, digit, cmd
+        compute         digit, _db and 0Fh
+        call            hex_nibble, digit, cmd
+        compute         i, i + 1
+        check           i = bytes
+        jyes            done
+        arrange         awk, cmd ','
+        assemble        awk
+        check           i and 0Fh
+        jyes            jj
+        arrange         awk, cmd '\',13,10
+        assemble        awk
+        jump            jj
+done:   arrange         awk, cmd 13,10
+        assemble        awk
+end calminstruction
+
+else
+        display 'usage: fasmg -i "define FILE ''<filename>''" ',__FILE__,' <outfile>'
+end if