init

Author: pfeilbr

.gitignore

@@ -0,0 +1,18 @@
+# aws-glue-pyspark-fetch-databases-and-tables-metadata-playground
+
+example AWS Glue pyspark job script that fetches all the catalog databases and tables metadata.
+
+* first method uses [spark sql](https://spark.apache.org/sql/)
+* second method uses [python boto3 Glue client](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue.html) to interact with Glue API directly
+
+see [`main.py`](main.py)
+
+## Notes
+
+ensure `--enable-glue-datacatalog` is enabled for glue job to allow spark sql to access metadata catalog
+
+![](https://www.evernote.com/l/AAG3O9zQGjhBQYiqT7_owkUm9K-UXd0bMCEB/image.png)
+
+Glue Console Script View
+
+![](https://www.evernote.com/l/AAG2b5Bdis5KFbt6ijxtySgIG7e2P8jPE0UB/image.png)

README.md

@@ -0,0 +1,53 @@
+import sys
+from awsglue.transforms import *
+from awsglue.utils import getResolvedOptions
+from pyspark.context import SparkContext
+from awsglue.context import GlueContext
+from awsglue.job import Job
+
+args = getResolvedOptions(sys.argv, ['JOB_NAME'])
+
+sc = SparkContext()
+glueContext = GlueContext(sc)
+spark = glueContext.spark_session
+job = Job(glueContext)
+
+# fetch database and table metadata
+
+# --- using spark sql example ---
+# NOTE: `--enable-glue-datacatalog` must be enabled for job
+def spark_sql_example():
+    spark.sql("use default")
+    tables = spark.sql("show tables").rdd.collect()
+    output = ''
+    for table in tables:
+        output += f"-- schema for {table.tableName} ---\n"
+        tableDescribe = spark.sql(f"describe `{table.tableName}`").rdd.collect()
+        for column in tableDescribe:
+            output += f"column name: {column['col_name']}, type: {column['data_type']}\n"
+        output +="\n\n\n"
+
+    print(output)
+
+# --- using boto3 example ---
+
+import boto3
+import json
+
+def glue_client_example():
+    client = boto3.client('glue')
+    databases_resp = client.get_databases()
+    for database in databases_resp['DatabaseList']:
+        database_name = database['Name']
+        tables_resp = client.get_tables(DatabaseName=database_name)
+        for table in tables_resp['TableList']:
+            table_name = table['Name']
+            table_resp = client.get_table(DatabaseName=database_name, Name=table_name)
+            print(json.dumps(table_resp, indent=2, sort_keys=True, default=str))
+
+def main():
+    spark_sql_example()
+    glue_client_example()
+
+main()
+