Awesome

AWS DMS Terraform module

Terraform module which creates AWS DMS (Database Migration Service) resources.

Usage

See examples directory for working examples to reference:

module "database_migration_service" {
  source  = "terraform-aws-modules/dms/aws"
  version = "~> 2.0"

  # Subnet group
  repl_subnet_group_name        = "example"
  repl_subnet_group_description = "DMS Subnet group"
  repl_subnet_group_subnet_ids  = ["subnet-1fe3d837", "subnet-129d66ab", "subnet-1211eef5"]

  # Instance
  repl_instance_allocated_storage            = 64
  repl_instance_auto_minor_version_upgrade   = true
  repl_instance_allow_major_version_upgrade  = true
  repl_instance_apply_immediately            = true
  repl_instance_engine_version               = "3.5.2"
  repl_instance_multi_az                     = true
  repl_instance_preferred_maintenance_window = "sun:10:30-sun:14:30"
  repl_instance_publicly_accessible          = false
  repl_instance_class                        = "dms.t3.large"
  repl_instance_id                           = "example"
  repl_instance_vpc_security_group_ids       = ["sg-12345678"]

  endpoints = {
    source = {
      database_name               = "example"
      endpoint_id                 = "example-source"
      endpoint_type               = "source"
      engine_name                 = "aurora-postgresql"
      extra_connection_attributes = "heartbeatFrequency=1;"
      username                    = "postgresqlUser"
      password                    = "youShouldPickABetterPassword123!"
      port                        = 5432
      server_name                 = "dms-ex-src.cluster-abcdefghijkl.us-east-1.rds.amazonaws.com"
      ssl_mode                    = "none"
      tags                        = { EndpointType = "source" }
    }

    destination = {
      database_name = "example"
      endpoint_id   = "example-destination"
      endpoint_type = "target"
      engine_name   = "aurora"
      username      = "mysqlUser"
      password      = "passwordsDoNotNeedToMatch789?"
      port          = 3306
      server_name   = "dms-ex-dest.cluster-abcdefghijkl.us-east-1.rds.amazonaws.com"
      ssl_mode      = "none"
      tags          = { EndpointType = "destination" }
    }
  }

  replication_tasks = {
    cdc_ex = {
      replication_task_id       = "example-cdc"
      migration_type            = "cdc"
      replication_task_settings = file("task_settings.json")
      table_mappings            = file("table_mappings.json")
      source_endpoint_key       = "source"
      target_endpoint_key       = "destination"
      tags                      = { Task = "PostgreSQL-to-MySQL" }
    }
  }

  event_subscriptions = {
    instance = {
      name                             = "instance-events"
      enabled                          = true
      instance_event_subscription_keys = ["example"]
      source_type                      = "replication-instance"
      sns_topic_arn                    = "arn:aws:sns:us-east-1:012345678910:example-topic"
      event_categories                 = [
        "failure",
        "creation",
        "deletion",
        "maintenance",
        "failover",
        "low storage",
        "configuration change"
      ]
    }
    task = {
      name                         = "task-events"
      enabled                      = true
      task_event_subscription_keys = ["cdc_ex"]
      source_type                  = "replication-task"
      sns_topic_arn                = "arn:aws:sns:us-east-1:012345678910:example-topic"
      event_categories             = [
        "failure",
        "state change",
        "creation",
        "deletion",
        "configuration change"
      ]
    }
  }

  tags = {
    Terraform   = "true"
    Environment = "dev"
  }
}

Combinations

Within DMS you can have multiple combinations of resources depending on your use case. For example (not an exhaustive list of possible combinations):

Simple

• One source endpoint
• One target/destination endpoint
• One replication task
• Two event subscriptions
  • Replication instance event subscription
  • Replication task event subscriptions

Multiple endpoints, multiple tasks

• Two source endpoints
• Three target/destination endpoints
• Three replication tasks (source1 -> target1, source2 -> target2, source1 -> targe3)
• Four event subscriptions
  • Replication instance event subscription
  • Replication task event subscription for each task listed above

In order to accommodate a flexible, multi-resource combinatorial module, keys and maps are used for cross-referencing resources created within the module.

Given the following example (not complete, only showing the relevant attributes):

module "database_migration_service" {
  source  = "terraform-aws-modules/dms/aws"
  version = "~> 2.0"

  # This value is used when subscribing to instance event notifications
  repl_instance_id = "readme-example"

  endpoints = {
    # These keys are used to map endpoints within task definitions by this key `source1`
    source1 = {
      endpoint_type = "source"
      ...
    }

    destination1 = {
      endpoint_type = "target"
      ...
    }

    destination2 = {
      endpoint_type = "target"
      ...
    }
  }

To create replication tasks, you simply reference the relevant keys from the endpoints map in the source_endpoint_key/target_endpoint_key fields:

  ...

  replication_tasks = {
    src1_dest1 = {
      ...
      source_endpoint_key = "source1"
      target_endpoint_key = "destination1"
    }
    src1_dest2 = {
      ...
      source_endpoint_key = "source1"
      target_endpoint_key = "destination2"
    }
  }

  ...

Continuing the same lookup pattern, to create event subscriptions, you simply reference the replication instance ID in the instance_event_subscription_keys field when subscribing to instance notifications, or the replication_tasks keys in the task_event_subscription_keys to subscribe to the tasks notifications (all or only select keys for select tasks):

  ...

  event_subscriptions = {
    instance = {
      instance_event_subscription_keys = ["readme-example"]
      ...
    }
    task = {
      task_event_subscription_keys = ["src1_dest1", "src1_dest2]
      ...
    }
  }

  ...

Tasks

Tasks are the "jobs" that perform the necessary actions of migrating from source to target, including any transformations and/or mappings of the data in transit. Tasks are largely controlled by task settings that are defined in a JSON document.

Example task settings JSON document:

{
  "TargetMetadata": {
    "TargetSchema": "",
    "SupportLobs": true,
    "FullLobMode": false,
    "LobChunkSize": 64,
    "LimitedSizeLobMode": true,
    "LobMaxSize": 32,
    "BatchApplyEnabled": true
  },
  "FullLoadSettings": {
    "TargetTablePrepMode": "DO_NOTHING",
    "CreatePkAfterFullLoad": false,
    "StopTaskCachedChangesApplied": false,
    "StopTaskCachedChangesNotApplied": false,
    "MaxFullLoadSubTasks": 8,
    "TransactionConsistencyTimeout": 600,
    "CommitRate": 10000
  },
  "Logging": {
    "EnableLogging": false
  },
  "ControlTablesSettings": {
    "ControlSchema":"",
    "HistoryTimeslotInMinutes":5,
    "HistoryTableEnabled": false,
    "SuspendedTablesTableEnabled": false,
    "StatusTableEnabled": false
  },
  "StreamBufferSettings": {
    "StreamBufferCount": 3,
    "StreamBufferSizeInMB": 8
  },
  "ChangeProcessingTuning": {
    "BatchApplyPreserveTransaction": true,
    "BatchApplyTimeoutMin": 1,
    "BatchApplyTimeoutMax": 30,
    "BatchApplyMemoryLimit": 500,
    "BatchSplitSize": 0,
    "MinTransactionSize": 1000,
    "CommitTimeout": 1,
    "MemoryLimitTotal": 1024,
    "MemoryKeepTime": 60,
    "StatementCacheSize": 50
  },
  "ChangeProcessingDdlHandlingPolicy": {
    "HandleSourceTableDropped": true,
    "HandleSourceTableTruncated": true,
    "HandleSourceTableAltered": true
  },
  "ErrorBehavior": {
    "DataErrorPolicy": "LOG_ERROR",
    "DataTruncationErrorPolicy":"LOG_ERROR",
    "DataErrorEscalationPolicy":"SUSPEND_TABLE",
    "DataErrorEscalationCount": 50,
    "TableErrorPolicy":"SUSPEND_TABLE",
    "TableErrorEscalationPolicy":"STOP_TASK",
    "TableErrorEscalationCount": 50,
    "RecoverableErrorCount": 0,
    "RecoverableErrorInterval": 5,
    "RecoverableErrorThrottling": true,
    "RecoverableErrorThrottlingMax": 1800,
    "ApplyErrorDeletePolicy":"IGNORE_RECORD",
    "ApplyErrorInsertPolicy":"LOG_ERROR",
    "ApplyErrorUpdatePolicy":"LOG_ERROR",
    "ApplyErrorEscalationPolicy":"LOG_ERROR",
    "ApplyErrorEscalationCount": 0,
    "FullLoadIgnoreConflicts": true
  }
}

Examples

Examples codified under the examples are intended to give users references for how to use the module(s) as well as testing/validating changes to the source code of the module. If contributing to the project, please be sure to make any appropriate updates to the relevant examples to allow maintainers to test your changes and to keep the examples up to date for users. Thank you!

• Complete
• Serverless

Requirements

Providers

Modules

No modules.

Resources

Inputs

Outputs

License

Apache-2.0 Licensed. See LICENSE.