Support literals encoding conversions according to the literal type.

ChangeLog

@@ -1,3 +1,12 @@
+2022-03-20  Nicolas Berthier <[email protected]>
+
+	* .github/workflows: improve CI setup
+
+2022-02-04  David Declerck <[email protected]>
+
+	* config, cobc/cobc.h: add support for GCOS 7 dialect (Bull), both
+	  a strict variant (config/gcos-strict.conf) and a variant with
+	  GNUCobol extensions (config/gcos.conf)
 
 2024-12-08  Simon Sobisch <[email protected]>
 
@@ -184,6 +193,10 @@
 
 	* configure.ac: dropped extra check for GCC as done internally
 
+2022-09-30  Nicolas Berthier <[email protected]>
+
+	* .github/workflows: fix ubuntu CI setup
+
 2022-09-08  Simon Sobisch <[email protected]>
 
 	* configure.ac: cleanup curses library check
@@ -201,6 +214,10 @@
 
 	* configure.ac: check for PDC_free_memory_allocations
 
+2022-07-20  Nicolas Berthier <[email protected]>
+
+	* .github/workflows: fix handling of quotes in testuite artifact name
+
 2022-07-07  Simon Sobisch <[email protected]>
 
 	* .github/workflows: CI now emits a coverage report artifact
@@ -239,7 +256,7 @@
 2022-03-29  Simon Sobisch <[email protected]>
 
 	* configure.ac: dropped obsolete AC_PROG_CC_STD, AC_HEADER_STDC as
-	  already included in AC_PROG_CC and assumed otherwise
+          already included in AC_PROG_CC and assumed otherwise
 
 2022-03-11  Simon Sobisch <[email protected]>
 

NEWS

@@ -6,6 +6,9 @@ NEWS - user visible changes				-*- outline -*-
 
 * New GnuCOBOL features
 
+** support different literal encoding according to literal type (UTF-8 for U literals, 
+   UTF-16 for N literals, and ISO-8859-15 for Alphanumeric literals)
+
 ** cobc now checks for binary and multi-byte encoded files and early exit
    parsing those; the error output for format errors (for example invalid
    indicator column) is now limitted to 5 per source file

THANKS

@@ -110,4 +110,4 @@ David Essex
 Jeff Smith
 Jim Noeth
 Stephen Connolly
-Laura Tweedy
+Laura Tweedy

cobc/ChangeLog

@@ -52,6 +52,12 @@
 	* typeck.c (cb_emit_move, cb_emit_set_to): do not check for incompatible
 	  data if no receiver field is of category numeric or numeric edited
 
+2024-09-09  Simon Sobisch <[email protected]>
+
+	* cobc.c, codegen.c, tree.c: fixed C89 errors
+	* typeck.c (trimmed_size, is_blank): minor refactorings
+	* typeck.c: provide the original buffer if encoding cannot be applied
+
 2024-09-03  Simon Sobisch <[email protected]>
 
 	* typeck.c (cb_emit_accept): always check position
@@ -76,6 +82,21 @@
 	* tree.c (char_to_precedence_idx, get_char_type_description, valid_char_order):
 	  adjusted size of precedence table and gave proper precedence to U
 
+2024-08-27  Simon Sobisch <[email protected]>
+
+	* cobc.c, codegen.c, tree.c: fixed C89 errors
+	* typeck.c (trimmed_size, is_blank): minor refactorings
+
+2024-08-10  Ahmed Maher <[email protected]>
+
+	* cobc.c: added the locale option to specify the source 
+	  file encoding if not specified with command line argument 
+
+2024-08-10  Ahmed Maher <[email protected]>
+
+	* cobc.h, cobc.c, flag.def: added command line argument for setting
+	  source file encoding
+
 2024-08-06  Simon Sobisch <[email protected]>
 
 	* codegen.c (output_alphabet_name_definition): cater for national alphabet
@@ -89,6 +110,14 @@
 	  national alphabets with increased size (max. 65535 instead of 255)
 	* typeck.c (validate_alphabet): check that alphabet and literal types match
 
+
+2024-08-1 Ahmed Maher<[email protected]>
+
+	* scanner.l, cobc.h, cobc.c, tree.c: added guards around iconv.h header and
+	  used it around the usages of the headers functions
+	* tree.c (cb_build_utf8_literal): new function for UTF-8 literals
+	* scanner.l (read_literal): changed to handle the utf8 literal reading 
+
 2024-07-29  Chuck Haatvedt <[email protected]>
 
 	* tree.c (cb_build_picture): added logic to find the valid floating
@@ -103,6 +132,10 @@
 	* tree.c (char_to_precedence_idx): changed to check penultimate and last
 	  symbols before the first and second symbols
 
+2024-07-27 Ahmed Maher <[email protected]>
+
+	* scanner.l: refactor scan_x function
+
 2024-07-26  Simon Sobisch <[email protected]>
 
 	* parser.y, config.def: rewrite ENVIRONMENT DIVISION parsing to allow
@@ -111,10 +144,23 @@
 	* reserved.c: make MENU context-sensitive
 	* reserved.c, parser.y: added MODAL + MODELESS to acu extension windows
 
+
 2024-07-10  Chuck Haatvedt <[email protected]>
 
 	* tree.c (cb_build_picture): fixed currency scale counting logic
 
+2024-07-09  Ahmed Maher <[email protected]>
+            James K. Lowden  <[email protected]>
+
+	* typeck.c: refactor validate_move function
+
+2024-07-06 Ahmed Maher <[email protected]>
+
+	* tree.c: added encoding conversion support to cb_build_national_literal and
+	cb_build_alphanumeric_literal
+	* cobc.h: declared struct cb_iconv used for encoding conversion
+	* cobc.c: defined the struct cb_iconv with default values
+
 2024-06-19  David Declerck <[email protected]>
 
 	* cobc.c (process_compile): fix MSVC build command

cobc/cobc.c

@@ -52,6 +52,11 @@
 #include <io.h>
 #endif
 
+
+#ifdef	HAVE_LANGINFO_CODESET
+#include <langinfo.h>
+#endif
+
 #ifdef	HAVE_LOCALE_H
 #include <locale.h>
 #endif
@@ -116,6 +121,9 @@ enum compile_level {
 #define CB_FLAG_GETOPT_DEPEND_ADD_PHONY     24
 #define CB_FLAG_GETOPT_DEPEND_KEEP_MISSING  25
 #define CB_FLAG_GETOPT_DEPEND_ON_THE_SIDE   26
+#define CB_FLAG_GETOPT_SOURCE_ENCODE        27
+#define CB_FLAG_GETOPT_ALPHANUMERIC_ENCODE  28
+
 
 /* Info display limits */
 #define	CB_IMSG_SIZE		24
@@ -327,6 +335,39 @@ cob_u32_t		optimize_defs[COB_OPTIM_MAX] = { 0 };
 
 int cb_flag_alt_ebcdic = 0;
 
+#ifdef HAVE_ICONV
+struct cb_iconv_t cb_iconv;
+
+static void 
+initialize_cb_iconv() {
+#ifdef HAVE_LANGINFO_CODESET
+	char * encoding;
+	char * locale = setlocale (LC_CTYPE, "");
+	if( ! locale  ) {
+		cobc_err_msg ("could not set locale");
+		return;
+  	}
+	encoding = nl_langinfo(CODESET);
+
+	if (encoding != NULL && *encoding > 0) {
+		strncpy (cb_iconv.source, encoding, sizeof(cb_iconv.source) - 1);
+		cb_iconv.source[sizeof(cb_iconv.source) - 1] = '\0';
+	} else {
+		strncpy (cb_iconv.source, "ISO-8859-15", sizeof(cb_iconv.source) - 1);
+		cb_iconv.source[sizeof(cb_iconv.source) - 1] = '\0';
+	}
+#else
+	strncpy(cb_iconv.source, "ISO-8859-15", sizeof(cb_iconv.source) - 1);
+    cb_iconv.source[sizeof(cb_iconv.source) - 1] = '\0';
+#endif
+	/* set the alphanumeric_source encoding to a default value 
+	to avoid converting in cb_build_alphanumeric 
+	if it didn't change by the command line*/
+	strncpy (cb_iconv.alphanumeric_source, "NONE", sizeof(cb_iconv.alphanumeric_source) - 1);
+	cb_iconv.alphanumeric_source[sizeof(cb_iconv.alphanumeric_source) - 1] = '\0';
+}
+#endif
+
 
 /* Basic memory structure */
 struct cobc_mem_struct {
@@ -3959,6 +4000,62 @@ process_command_line (const int argc, char **argv)
 			}
 			break;
 
+		case CB_FLAG_GETOPT_SOURCE_ENCODE:
+			/* -fsource-encode=encoding*/
+			const char* valid_encodings[] = {
+				"UTF-8",
+				"UTF8",
+				"ASCII",
+				"ISO-8859-1",
+				"ISO-8859-15",
+				"CP1525"
+			};
+			const int num_encodings = sizeof(valid_encodings) / sizeof(valid_encodings[0]);
+			int i, encoding_valid = 0;
+			for (i = 0; i < num_encodings; i++) {
+				if (strcmp(cob_optarg, valid_encodings[i]) == 0) {
+					encoding_valid = 1;
+					break;
+				}
+			}
+			if (encoding_valid) {
+#ifdef HAVE_ICONV
+				strncpy(cb_iconv.source, cob_optarg, sizeof(cb_iconv.source) - 1);
+				cb_iconv.source[sizeof(cb_iconv.source) - 1] = '\0';
+#endif
+			} else {
+				cobc_err_exit(COBC_INV_PAR, "-fsource-encode");
+			}
+			break;
+
+
+		/* -falphanumeric-encode */
+		case CB_FLAG_GETOPT_ALPHANUMERIC_ENCODE:{
+			const char* valid_encodings[] = {
+				"ASCII",
+				"ISO-8859-1",
+				"ISO-8859-15",
+				"CP1525"
+			};
+			const int num_encodings = sizeof(valid_encodings) / sizeof(valid_encodings[0]);
+			int i, encoding_valid = 0;
+			for (i = 0; i < num_encodings; i++) {
+				if (strcmp(cob_optarg, valid_encodings[i]) == 0) {
+					encoding_valid = 1;
+					break;
+				}
+			}
+			if (encoding_valid) {
+#ifdef HAVE_ICONV
+				strncpy(cb_iconv.alphanumeric_source, cob_optarg, sizeof(cb_iconv.alphanumeric_source) - 1);
+				cb_iconv.alphanumeric_source[sizeof(cb_iconv.alphanumeric_source) - 1] = '\0';
+#endif
+			} else {
+				cobc_err_exit(COBC_INV_PAR, "-falphanumeric-encode");
+			}
+			break;
+		}
+
 		case CB_FLAG_GETOPT_TTITLE: {
 			/* -fttitle=<title> : Title for listing */
 			const size_t len = strlen (cob_optarg);
@@ -9139,7 +9236,12 @@ static void
 begin_setup_internal_and_compiler_env (void)
 {
 	char			*p;
-
+
+	/* initialize the default source encoding */
+#ifdef HAVE_ICONV
+	initialize_cb_iconv();
+#endif
+
 	/* register signal handlers from cobc */
 	cob_reg_sighnd (&cobc_sig_handler);
 
@@ -9557,6 +9659,16 @@ main (int argc, char **argv)
 	cobc_init_tree ();
 #endif
 
+/* initialize the iconv struct after reading the command line*/
+#ifdef HAVE_ICONV
+	cb_iconv.alphanumeric = iconv_open(cb_iconv.alphanumeric_source, cb_iconv.source);
+	/* move iconv_open check here */
+
+	cb_iconv.national = iconv_open("UTF-16LE", cb_iconv.source);
+	cb_iconv.utf8 = iconv_open("UTF-8", cb_iconv.source);
+#endif
+
+
 	/* Process input files */
 
 	/* Set up file parameters, if any are missing: abort */

cobc/cobc.h

@@ -30,6 +30,13 @@
 #include <unistd.h>
 #endif
 #include <stdio.h>	/* for FILE* */
+
+#ifdef HAVE_ICONV
+#include <iconv.h>
+#endif
+
+#include <errno.h>
+
 
 #include "../libcob/common.h"
 
@@ -322,6 +329,21 @@ enum cobc_name_type {
 	PROGRAM_ID_NAME
 };
 
+
+#ifdef HAVE_ICONV
+/* Structure for a iconv conversion */
+struct cb_iconv_t{
+    iconv_t alphanumeric;
+    iconv_t national;
+	iconv_t utf8;
+	char source[100];
+	char alphanumeric_source[100];
+};
+
+extern struct cb_iconv_t cb_iconv;
+#endif
+
+
 /* Listing structures and externals */
 
 #if 0 /* ancient OSVS registers that need special runtime handling - low priority */

cobc/codegen.c

@@ -5205,7 +5205,24 @@ output_initialize_to_value (struct cb_field *f, cb_tree x,
 			   from a string - we use a local buffer to set that up */
 			unsigned char litbuff[128];
 			memcpy (litbuff + litstart, l->data, l->size);
-			memset (litbuff + padstart, ' ', padlen);
+
+			if (l->common.category == CB_CATEGORY_NATIONAL || l->common.category == CB_CATEGORY_NATIONAL_EDITED){
+				/* Calculate the number of bytes to pad (multiply by 2 for UTF-16) */
+				const size_t padbytes = padlen * 2;
+				size_t i;
+
+				/* Get a pointer to the starting position for padding */
+				unsigned char *padptr = litbuff + padstart;
+
+				/* Pad the rest of the string with UTF-16 space character */
+				for (i = 0; i < padbytes; i += 2) {
+					padptr[i] = 0x20;    
+					padptr[i + 1] = 0x00; 
+				}
+			} else{
+				memset (litbuff + padstart, ' ', padlen);
+			}
+
 			output_prefix ();
 			output ("memcpy (");
 			output_data (x);
@@ -5247,7 +5264,23 @@ output_initialize_to_value (struct cb_field *f, cb_tree x,
 		}
 
 		memcpy (litbuff + litstart, l->data, l->size);
-		memset (litbuff + padstart, ' ', padlen);
+
+		if(x->category == CB_CATEGORY_NATIONAL || x->category == CB_CATEGORY_NATIONAL_EDITED){
+			// Calculate the number of bytes to pad (multiply by 2 for UTF-16)
+			size_t padbytes = padlen * 2;
+
+			// Get a pointer to the starting position for padding
+			unsigned char *padptr = litbuff + padstart;
+
+			// Pad the rest of the string with UTF-16 space character
+			for (size_t i = 0; i < padbytes; i += 2) {
+				padptr[i] = 0x20;    
+				padptr[i + 1] = 0x00; 
+			}
+		}
+		else{
+			memset (litbuff + padstart, ' ', padlen);
+		}
 
 		buffchar = *(litbuff + size - 1);
 		n = 0;