AWSTemplateFormatVersion: '2010-09-09'
Description: 'CloudFormation template for AWS Glue jobs, AWS Glue databases, Amazon S3 table buckets, Amazon S3 table namespaces, and associated AWS Identity and Access Management roles'

Parameters:
  ScriptBucketName:
    Type: String
    Description: Name of the referenced Amazon S3 general purpose bucket containing the AWS Glue job scripts
  IcebergScriptPath:
    Type: String
    Description: The referenced Amazon S3 path to the AWS Glue script file for the Apache Iceberg table creation job
    Default: scripts/create_iceberg_table_on_s3.py
  S3TableScriptPath:
    Type: String
    Description: The referenced Amazon S3 path to the AWS Glue script file for the Amazon S3 table creation job
    Default: scripts/create_s3_table_on_s3_bucket.py
  OutputBucketName:
    Type: String
    Description: Name of the created Amazon S3 general purpose bucket for the AWS Glue job for Apache Iceberg table data
  InputS3Bucket:
    Type: String
    Description: Name of the referenced Amazon S3 bucket with which the NY Taxi data was uploaded
  InputS3Path:
    Type: String
    Default: nyc_yellow_trip_data
    Description: The referenced Amazon S3 path with which the NY Taxi data was uploaded
  GlueIcebergJobName:
    Type: String
    Description: The name of the created AWS Glue job that creates the Apache Iceberg table on general purpose Amazon S3 bucket
    Default: create_iceberg_table_on_s3
  GlueS3TableJobName:
    Type: String
    Description: The name of the created AWS Glue job that creates the Amazon S3 table
    Default: create_s3_table_on_s3_bucket
  DatabaseName:
    Type: String
    Description: Name of the AWS Glue Database to be created for the Apache Iceberg table on general purpose Amazon S3 bucket
    Default: iceberg_dq_demo
    MinLength: 1
    MaxLength: 255
    AllowedPattern: '[a-zA-Z0-9_]*'
    ConstraintDescription: Database name must contain only alphanumeric characters and underscores
  S3TableBucketName:
    Type: String
    Description: Name of the Amazon S3 table bucket to be created. 
    Default: dataquality-demo-bucket
    MinLength: 3
    MaxLength: 63
    AllowedPattern: '[a-z0-9-]*'
    ConstraintDescription: See our documentation for bucket naming rules https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-tables-buckets-naming.html
  S3TableNamespaceName:
    Type: String
    Description: Name of the Amazon S3 table bucket namespace to be created
    MinLength: 1
    MaxLength: 255
    AllowedPattern: '[a-z0-9_]*'
    ConstraintDescription: Namespace name must contain only lowercase alphanumeric characters and underscores. See more information here https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-tables-namespace-create.html
    Default: s3_table_dq_demo
  S3TableTableName:
    Type: String
    Description: Name of the Amaazon S3 table to be created by the AWS Glue job
    Default: ny_taxi
    MinLength: 1
    MaxLength: 255
    AllowedPattern: '[a-z0-9_]*'
    ConstraintDescription: Table name must only consist of lowercase alphanumeric characters and underscores. See more information here https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-tables-namespace-create.html
  IcebergTableName:
    Type: String
    Description: Name of the Apache Iceberg table on general purpose Amazon S3 to be created by the AWS Glue job
    Default: ny_taxi
    MinLength: 1
    MaxLength: 255
    AllowedPattern: '[a-z0-9_]*'
    ConstraintDescription: Table name must only consist of lowercase alphanumeric characters and underscores. See more information here https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-tables.html#aws-glue-api-catalog-tables-Table


Resources:
  # S3 Output Bucket
  OutputBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !Ref OutputBucketName
      VersioningConfiguration:
        Status: Enabled
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true

  DataQualityRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: glue.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole
      Policies:
        - PolicyName: LFandS3TableAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - lakeformation:GetDataAccess
                Resource: 
                  - '*'
              - Effect: Allow
                Action:
                  - s3tables:GetTableData
                  - s3tables:GetTableMetadataLocation
                Resource:
                  - !Sub arn:aws:s3tables:${AWS::Region}:${AWS::AccountId}:bucket/${S3TableBucketName}/table/*
  GlueDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: !Ref DatabaseName
        Description: 'Glue Database created via CloudFormation'
        LocationUri: !Sub s3://${OutputBucketName}
  LakeFormationRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lakeformation.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: S3Access
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:PutObject
                  - s3:GetObject
                  - s3:DeleteObject
                Resource: !Sub arn:aws:s3:::${OutputBucketName}/*
              - Effect : Allow 
                Action:
                  - s3:ListBucket
                Resource: !Sub arn:aws:s3:::${OutputBucketName}
              - Effect : Allow
                Action: 
                  - s3:ListAllMyBuckets
                Resource: arn:aws:s3:::*
  # IAM Role for Glue Job
  GlueJobRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: glue.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole
      Policies:
        - PolicyName: S3andS3TableAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:PutObject
                  - s3:GetObject
                  - s3:DeleteObject
                Resource: !Sub arn:aws:s3:::${OutputBucketName}/*
              - Effect : Allow
                Action: 
                  - s3:GetObject
                Resource: !Sub arn:aws:s3:::${InputS3Bucket}/*
              - Effect : Allow 
                Action:
                  - s3:ListBucket
                Resource: !Sub arn:aws:s3:::${InputS3Bucket}
              - Effect : Allow
                Action: 
                  - s3:GetObject
                Resource: !Sub arn:aws:s3:::${ScriptBucketName}/${IcebergScriptPath}
              - Effect : Allow
                Action: 
                  - s3:GetObject
                Resource: !Sub arn:aws:s3:::${ScriptBucketName}/${S3TableScriptPath}
              - Effect : Allow 
                Action:
                  - s3:ListBucket
                Resource: !Sub arn:aws:s3:::${ScriptBucketName}
              - Effect: Allow
                Action: 
                  - s3tables:GetTablePolicy
                  - s3tables:GetTable
                  - s3tables:GetTableEncryption
                  - s3tables:CreateTable
                  - s3tables:GetNamespace
                  - s3tables:PutTableData
                  - s3tables:GetTableBucketEncryption
                  - s3tables:GetTableBucketMaintenanceConfiguration
                  - s3tables:GetTableMaintenanceJobStatus
                  - s3tables:GetTableMetadataLocation
                  - s3tables:UpdateTableMetadataLocation
                  - s3tables:GetTableData
                  - s3tables:GetTableBucket
                  - s3tables:GetTableMaintenanceConfiguration
                  - s3tables:GetTableBucketPolicy
                Resource:
                  - '*'
  S3TableBucket:
    Type: AWS::S3Tables::TableBucket
    Properties:
      TableBucketName: !Ref S3TableBucketName
      UnreferencedFileRemoval: 
        NoncurrentDays: 10
        Status: Enabled
        UnreferencedDays: 3

  S3TableBucketNamespace:
    Type: AWS::S3Tables::Namespace
    DependsOn: S3TableBucket
    Properties:
      Namespace: !Ref S3TableNamespaceName
      TableBucketARN: !Sub arn:aws:s3tables:${AWS::Region}:${AWS::AccountId}:bucket/${S3TableBucketName}
  # Glue Job
  GlueIcebergJob:
    Type: AWS::Glue::Job
    Properties:
      Name: !Ref GlueIcebergJobName
      Role: !GetAtt GlueJobRole.Arn
      Command:
        Name: glueetl
        ScriptLocation: !Sub s3://${ScriptBucketName}/${IcebergScriptPath}
      DefaultArguments:
        '--job-language': python
        '--output_bucket': !Ref OutputBucketName
        '--datalake-formats': 'iceberg'
        '--input_data_path': !Sub 's3://${InputS3Bucket}/${InputS3Path}'
        '--database': !Ref DatabaseName
        '--table': !Ref IcebergTableName
        '--job-bookmark-option': job-bookmark-disable
      MaxRetries: 0
      Timeout: 60
      WorkerType: 'G.1X'
      GlueVersion: '5.0'
      NumberOfWorkers: 2  
  GlueS3TableJob:
    Type: AWS::Glue::Job
    DependsOn: S3TableBucket
    Properties:
      Name: !Ref GlueS3TableJobName
      Role: !GetAtt GlueJobRole.Arn
      Command:
        Name: glueetl
        ScriptLocation: !Sub s3://${ScriptBucketName}/${S3TableScriptPath}
      DefaultArguments:
        '--region': !Ref AWS::Region
        '--job-language': python
        '--s3_bucket_arn': !Sub arn:aws:s3tables:${AWS::Region}:${AWS::AccountId}:bucket/${S3TableBucketName}
        '--input_data_path': !Sub 's3://${InputS3Bucket}/${InputS3Path}'
        '--namespace': !Ref S3TableNamespaceName
        '--job-bookmark-option': job-bookmark-disable
        '--s3_table_name': !Ref S3TableTableName
      MaxRetries: 0
      Timeout: 60
      WorkerType: 'G.1X'
      GlueVersion: '5.0'
      NumberOfWorkers: 2 