diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 2f96c96..a463874 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -9,9 +9,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do @type t :: %__MODULE__{ bucket: String.t(), key: String.t(), - region: String.t() + region: String.t(), + failover_regions: [String.t()] | nil } - defstruct [:bucket, :key, :region] + defstruct [:bucket, :key, :region, :failover_regions] @opts_schema [ bucket: [ @@ -28,6 +29,12 @@ defmodule RemotePersistentTerm.Fetcher.S3 do type: :string, required: true, doc: "The AWS region of the s3 bucket." + ], + failover_regions: [ + type: {:list, :string}, + required: false, + doc: + "A list of AWS regions to use if calls to the default region fail. They will be tried in order." ] ] @@ -50,7 +57,8 @@ defmodule RemotePersistentTerm.Fetcher.S3 do %__MODULE__{ bucket: valid_opts[:bucket], key: valid_opts[:key], - region: valid_opts[:region] + region: valid_opts[:region], + failover_regions: valid_opts[:failover_regions] }} end end @@ -60,7 +68,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do with {:ok, versions} <- list_object_versions(state), {:ok, %{etag: etag, version_id: version}} <- find_latest(versions) do Logger.info( - "found latest version of s3://#{state.bucket}/#{state.key}: #{etag} with version: #{version}" + bucket: state.bucket, + key: state.key, + version: version, + message: "Found latest version of object" ) {:ok, etag} @@ -72,17 +83,32 @@ defmodule RemotePersistentTerm.Fetcher.S3 do {:error, "could not find s3://#{state.bucket}/#{state.key}"} {:error, reason} -> - Logger.error("#{__MODULE__} - unknown error: #{inspect(reason)}") + Logger.error(%{ + bucket: state.bucket, + key: state.key, + reason: inspect(reason), + message: "Failed to get current version of object - unknown reason" + }) + {:error, "Unknown error"} end end @impl true def download(state) do - Logger.info("downloading s3://#{state.bucket}/#{state.key}...") + Logger.info( + bucket: state.bucket, + key: state.key, + message: "Downloading object from S3" + ) with {:ok, %{body: body}} <- get_object(state) do - Logger.debug("downloaded s3://#{state.bucket}/#{state.key}!") + Logger.debug( + bucket: state.bucket, + key: state.key, + message: "Downloaded object from S3" + ) + {:ok, body} else {:error, reason} -> @@ -94,7 +120,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do res = state.bucket |> ExAws.S3.get_bucket_object_versions(prefix: state.key) - |> aws_client_request(state.region) + |> aws_client_request(state) with {:ok, %{body: %{versions: versions}}} <- res do {:ok, versions} @@ -104,7 +130,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp get_object(state) do state.bucket |> ExAws.S3.get_object(state.key) - |> aws_client_request(state.region) + |> aws_client_request(state) end defp find_latest([_ | _] = contents) do @@ -123,8 +149,57 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp find_latest(_), do: {:error, :not_found} - defp aws_client_request(op, region) do - client().request(op, region: region) + defp aws_client_request(op, %{region: region, failover_regions: nil}), + do: client().request(op, region: region) + + defp aws_client_request( + op, + %{ + region: region, + bucket: bucket, + key: key, + failover_regions: failover_regions + } = state + ) + when is_list(failover_regions) do + with {:error, reason} <- client().request(op, region: region) do + Logger.error(%{ + bucket: bucket, + key: key, + region: region, + reason: inspect(reason), + message: "Failed to fetch from primary region, attempting failover regions" + }) + + try_failover_regions(op, failover_regions, state) + end + end + + defp try_failover_regions(_op, [], _state), do: {:error, "All regions failed"} + + defp try_failover_regions(op, [region | remaining_regions], state) do + Logger.info(%{ + bucket: state.bucket, + key: state.key, + region: region, + message: "Trying failover region" + }) + + case client().request(op, region: region) do + {:ok, result} -> + {:ok, result} + + {:error, reason} -> + Logger.error(%{ + bucket: state.bucket, + key: state.key, + region: region, + reason: inspect(reason), + message: "Failed to fetch from failover region" + }) + + try_failover_regions(op, remaining_regions, state) + end end defp client, do: Application.get_env(:remote_persistent_term, :aws_client, ExAws) diff --git a/mix.exs b/mix.exs index 75d1aaa..864b6b5 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule RemotePersistentTerm.MixProject do use Mix.Project @name "RemotePersistentTerm" - @version "0.10.1" + @version "0.11.0" @repo_url "https://github.com/AppMonet/remote_persistent_term" def project do diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index 6226d5d..f323997 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -5,15 +5,27 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do setup :verify_on_exit! import ExUnit.CaptureLog + @bucket "test-bucket" + @key "test-key" + @region "test-region" + @failover_regions ["failover-region-1", "failover-region-2"] + @version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4" + test "Unknown error returns an error for current_version/1" do expect(AwsClientMock, :request, fn _op, _opts -> {:error, :unknown_error} end) - assert capture_log(fn -> - assert {:error, "Unknown error"} = S3.current_version(%S3{bucket: "bucket"}) - end) =~ - "Elixir.RemotePersistentTerm.Fetcher.S3 - unknown error: :unknown_error" + log = + capture_log(fn -> + assert {:error, "Unknown error"} = + S3.current_version(%S3{bucket: "bucket", key: "key"}) + end) + + assert log =~ "bucket: \"bucket\"" + assert log =~ "key: \"key\"" + assert log =~ "reason: \":unknown_error\"" + assert log =~ "Failed to get current version of object - unknown reason" end describe "init/1" do @@ -26,4 +38,166 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do S3.init(bucket: bucket, key: key, region: region) end end + + describe "failover_regions" do + test "current_identifiers/1 tries first failover region when primary region fails" do + # Setup state with failover regions + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_regions: @failover_regions + } + + # Mock the AWS client to fail for primary region but succeed for first failover region + expect(AwsClientMock, :request, 2, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: "failover-region-1"] -> + {:ok, + %{ + body: %{ + versions: [ + %{version_id: @version, etag: "current-etag", is_latest: "true"} + ] + } + }} + end + end) + + log = + capture_log(fn -> + result = S3.current_version(state) + assert {:ok, "current-etag"} = result + end) + + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "Found latest version of object" + end + + test "download/1 tries first failover region when primary region fails" do + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_regions: @failover_regions + } + + # Mock the AWS client to fail for primary region but succeed for first failover region + expect(AwsClientMock, :request, 2, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: "failover-region-1"] -> + {:ok, %{body: "content from failover region"}} + end + end) + + log = + capture_log(fn -> + result = S3.download(state) + assert {:ok, "content from failover region"} = result + end) + + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "Downloading object from S3" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "Downloaded object from S3" + end + + test "returns error when primary and all failover regions fail" do + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_regions: @failover_regions + } + + # Mock the AWS client to fail for all regions + expect(AwsClientMock, :request, 3, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: "failover-region-1"] -> + {:error, "First failover region connection error"} + + [region: "failover-region-2"] -> + {:error, "Second failover region connection error"} + end + end) + + log = + capture_log(fn -> + result = S3.download(state) + assert {:error, message} = result + assert message =~ "All regions failed" + end) + + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "Downloading object from S3" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "reason: \"\\\"First failover region connection error\\\"\"" + assert log =~ "Failed to fetch from failover region" + assert log =~ "region: \"failover-region-2\"" + assert log =~ "reason: \"\\\"Second failover region connection error\\\"\"" + end + + test "tries second failover region when first failover region fails" do + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_regions: @failover_regions + } + + # Mock the AWS client to fail for primary and first failover region but succeed for second failover region + expect(AwsClientMock, :request, 3, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: "failover-region-1"] -> + {:error, "First failover region connection error"} + + [region: "failover-region-2"] -> + {:ok, %{body: "content from second failover region"}} + end + end) + + log = + capture_log(fn -> + result = S3.download(state) + assert {:ok, "content from second failover region"} = result + end) + + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "Downloading object from S3" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "reason: \"\\\"First failover region connection error\\\"\"" + assert log =~ "Failed to fetch from failover region" + assert log =~ "region: \"failover-region-2\"" + assert log =~ "Downloaded object from S3" + end + end end