From b1db34ae1b738768411e11c9aa89eb9ccbd6fbe0 Mon Sep 17 00:00:00 2001 From: Zach Daniel Date: Mon, 4 Jun 2018 02:31:51 -0400 Subject: [PATCH] abstracting away most behavior from adapters, enabling storage strategies (#53) --- README.md | 6 +- config/test.exs | 2 +- lib/adapters/adapter.ex | 22 +-- lib/adapters/datadog.ex | 347 +-------------------------------- lib/datadog/api_server.ex | 10 +- lib/datadog/utils.ex | 23 --- lib/span.ex | 13 +- lib/spandex.ex | 282 +++++++++++++++++++++------ lib/strategy.ex | 12 ++ lib/strategy/pdict.ex | 27 +++ lib/trace.ex | 3 +- lib/tracer.ex | 120 +++++++----- mix.exs | 2 +- test/plug/add_context_test.exs | 11 +- test/plug/end_trace_test.exs | 4 +- 15 files changed, 379 insertions(+), 505 deletions(-) delete mode 100644 lib/datadog/utils.ex create mode 100644 lib/strategy.ex create mode 100644 lib/strategy/pdict.ex diff --git a/README.md b/README.md index 7304242..9c092d9 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ contained as keys for that struct. They illustrate the keys that are known to ei keys or to have UI sugar with certain clients. Its hard to find any kind of list of these published anywhere, so let me know if you know of more! Examples -``` +```elixir Spandex.update_span(type: :db, http: [url: "/posts", status_code: 400], sql_query: [query: "SELECT * FROM posts", rows: 10]) ``` @@ -151,6 +151,10 @@ Spandex used to ship with function decorators, but those decorators had a habit The current trace_id and span_id can be retrieved with `Tracer.current_trace_id()` and `Tracer.current_span_id()`. This can then be used as `Tracer.continue_trace("new_trace", trace_id, span_id)`. New spans can then be logged from there and will be sent in a separate batch. +## Strategies + +There is (currently and temporarily) only one storage strategy, which can be changed via the `strategy` option. See tracer opt documentation for an example of setting it. To implement your own (ETS adapter should be on its way) simply implement the `Spandex.Strategy` behaviour. Keep in mind that the strategy is not an atomic pattern. It represents retrieving and wholesale replacing a trace, meaning that it is *not* safe to use across processes or concurrently. Each process should have its own store for its own generated spans. This should be fine because you can send multiple batches of spans for the same trace separately. + ## Datadog Api Sender Performance Originally, the library had an api server and spans were sent via `GenServer.cast`, but we've seen the need to introduce backpressure, and limit the overall amount of requests made. As such, the datadog api sender accepts `batch_size` and `sync_threshold` options. diff --git a/config/test.exs b/config/test.exs index 1685c33..a965e5a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -9,8 +9,8 @@ config :logger, :console, config :spandex, Spandex.Test.Support.Tracer, service: :spandex_test, adapter: Spandex.Adapters.Datadog, - env: "test", sender: Spandex.Test.DatadogTestApiServer, + env: "test", resource: "default", services: [ spandex_test: :db diff --git a/lib/adapters/adapter.ex b/lib/adapters/adapter.ex index 4dd34ed..49680b7 100644 --- a/lib/adapters/adapter.ex +++ b/lib/adapters/adapter.ex @@ -1,21 +1,11 @@ defmodule Spandex.Adapters.Adapter do @moduledoc """ - Describes the callback for a tracing adapter. Can be used to provide different - implementations of reporting/aggregating spans while still using the spandex - internal implementation. + The callbacks required to implement an adapter. """ - @callback start_trace(String.t(), Keyword.t()) :: {:ok, term} | {:error, term} - @callback start_span(String.t(), Keyword.t()) :: {:ok, term} | {:error, term} - @callback update_span(Keyword.t()) :: :ok | {:error, term} - @callback finish_span(Keyword.t()) :: :ok | {:error, term} - @callback finish_trace(Keyword.t()) :: :ok | {:error, term} - @callback span_error(Exception.t(), Keyword.t()) :: :ok | {:error, term} - @callback current_trace_id(Keyword.t()) :: term | nil | {:error, term} - @callback current_span_id(Keyword.t()) :: term | nil | {:error, term} - @callback current_span(Keyword.t()) :: term | nil - @callback continue_trace(String.t(), term, term, Keyword.t()) :: {:ok, term} | {:error, term} - @callback continue_trace_from_span(String.t(), map, Keyword.t()) :: {:ok, term} | {:error, term} - @callback update_top_span(Keyword.t()) :: :ok | {:error, term} - @callback update_all_spans(Keyword.t()) :: :ok | {} + @callback distributed_context(Plug.Conn.t(), Keyword.t()) :: {:ok, term} | {:error, term} + @callback trace_id() :: term + @callback span_id() :: term + @callback now() :: term + @callback default_sender() :: module end diff --git a/lib/adapters/datadog.ex b/lib/adapters/datadog.ex index 705b253..359085d 100644 --- a/lib/adapters/datadog.ex +++ b/lib/adapters/datadog.ex @@ -5,341 +5,22 @@ defmodule Spandex.Adapters.Datadog do @behaviour Spandex.Adapters.Adapter - alias Spandex.Datadog.Utils - alias Spandex.Span - require Logger - @doc """ - Starts a trace context in process local storage. - """ - @impl Spandex.Adapters.Adapter - @spec start_trace(String.t(), opts :: Keyword.t()) :: {:ok, term} | {:error, term} - def start_trace(name, opts) do - if get_trace() do - _ = Logger.error("Tried to start a trace over top of another trace.") - {:error, :trace_running} - else - trace_id = Utils.next_id() - - span = - opts - |> Keyword.put_new(:name, name) - |> Keyword.put(:trace_id, trace_id) - |> Keyword.put(:start, Utils.now()) - |> Keyword.put(:id, Utils.next_id()) - |> Span.new() - - case span do - {:error, errors} -> - {:error, errors} - - span -> - Logger.metadata(trace_id: trace_id) - - put_trace(%{id: trace_id, stack: [span], spans: []}) - - {:ok, trace_id} - end - end - end - - @doc """ - Starts a span and adds it to the span stack. - """ - @impl Spandex.Adapters.Adapter - @spec start_span(String.t(), opts :: Keyword.t()) :: {:ok, term} | {:error, term} - def start_span(name, opts) do - trace = get_trace(:undefined) - - case trace do - :undefined -> - {:error, :no_trace_context} - - %{stack: [current_span | _]} -> - new_span = Span.child_of(current_span, name, Utils.next_id(), Utils.now(), opts) - - case new_span do - {:error, errors} -> - {:error, errors} - - new_span -> - put_trace(%{trace | stack: [new_span | trace.stack]}) - - Logger.metadata(span_id: new_span.id) - - {:ok, new_span.id} - end - - _ -> - new_span = - opts - |> Keyword.put_new(:name, name) - |> Keyword.put(:trace_id, trace.id) - |> Keyword.put(:start, Utils.now()) - |> Keyword.put(:id, Utils.next_id()) - |> Span.new() - - case new_span do - {:error, errors} -> - {:error, errors} - - new_span -> - put_trace(%{trace | stack: [new_span | trace.stack]}) - - Logger.metadata(span_id: new_span.id) - - {:ok, new_span.id} - end - end - end - - @doc """ - Updates a span according to the provided context. - See `Spandex.Span.update/2` for more information. - """ - @impl Spandex.Adapters.Adapter - @spec update_span(Keyword.t()) :: :ok | {:error, term} - def update_span(opts) do - trace = get_trace() - - if trace do - new_stack = - List.update_at(trace.stack, 0, fn span -> - case Span.update(span, opts) do - {:error, _} -> span - span -> span - end - end) - - put_trace(%{trace | stack: new_stack}) - - :ok - else - {:error, :no_trace_context} - end - end - - @doc """ - Updates the top level span with information. Useful for setting overal trace context - """ - @impl Spandex.Adapters.Adapter - @spec update_top_span(Keyword.t()) :: :ok | {:error, term} - def update_top_span(opts) do - trace = get_trace() - - if trace do - new_stack = - trace.stack - |> Enum.reverse() - |> List.update_at(0, fn span -> - case Span.update(span, opts) do - {:error, _} -> span - span -> span - end - end) - - top_span = List.first(new_stack) - - put_trace(%{trace | stack: Enum.reverse(new_stack)}) - - {:ok, top_span} - else - {:error, :no_trace_context} - end - end - - @doc """ - Updates all spans - """ - @impl Spandex.Adapters.Adapter - @spec update_all_spans(Keyword.t()) :: :ok | {} - def update_all_spans(opts) do - trace = get_trace() - - if trace do - new_stack = - Enum.map(trace.stack, fn span -> - case Span.update(span, opts) do - {:error, _} -> span - span -> span - end - end) - - new_spans = - Enum.map(trace.spans, fn span -> - case Span.update(span, opts) do - {:error, _} -> span - span -> span - end - end) - - put_trace(%{trace | stack: new_stack, spans: new_spans}) - - :ok - else - {:error, :no_trace_context} - end - end - - @doc """ - Completes the current span, moving it from the top of the span stack - to the list of completed spans. - """ - @impl Spandex.Adapters.Adapter - @spec finish_span(Keyword.t()) :: :ok | {:error, term} - def finish_span(_opts) do - trace = get_trace() - - cond do - is_nil(trace) -> - {:error, :no_trace_context} - - Enum.empty?(trace.stack) -> - {:error, :no_span_context} - - true -> - new_stack = tl(trace.stack) - - completed_span = - trace.stack - |> hd() - |> Span.update(completion_time: Utils.now()) - - put_trace(%{trace | stack: new_stack, spans: [completed_span | trace.spans]}) - - :ok - end - end + @max_id 9_223_372_036_854_775_807 - @doc """ - Sends the trace to datadog and clears out the current trace data - """ @impl Spandex.Adapters.Adapter - @spec finish_trace(opts :: Keyword.t()) :: :ok | {:error, :no_trace_context} - def finish_trace(opts) do - trace = get_trace() - sender = opts[:sender] - - if trace do - unfinished_spans = Enum.map(trace.stack, &Span.update(&1, completion_time: Utils.now())) - - trace.spans - |> Kernel.++(unfinished_spans) - |> Enum.map(fn span -> - Span.update(span, completion_time: Utils.now()) - end) - |> sender.send_spans() - - delete_trace() - - :ok - else - {:error, :no_trace_context} - end - end + def trace_id(), do: :rand.uniform(@max_id) - @doc """ - Gets the current trace id - """ @impl Spandex.Adapters.Adapter - @spec current_trace_id(Keyword.t()) :: term | nil | {:error, term} - def current_trace_id(_opts) do - %{id: id} = get_trace(%{id: nil}) - id - end + def span_id(), do: trace_id() - @doc """ - Gets the current span id - """ @impl Spandex.Adapters.Adapter - @spec current_span_id(Keyword.t()) :: term | nil | {:error, term} - def current_span_id(_opts) do - case get_trace() do - %{stack: [%{id: current_span_id} | _]} -> - current_span_id + def now(), do: :os.system_time(:nano_seconds) - _ -> - nil - end - end - - @doc """ - Gets the current span - """ @impl Spandex.Adapters.Adapter - @spec current_span(Keyword.t()) :: term | nil | {:error, term} - def current_span(_opts) do - case get_trace() do - %{stack: [span | _]} -> - span - - _ -> - nil - end - end - - @doc """ - Continues a trace given a name and a span - """ - @impl Spandex.Adapters.Adapter - @spec continue_trace_from_span(String.t(), term, Keyword.t()) :: {:ok, term} - def continue_trace_from_span(name, %{trace_id: trace_id} = span, opts) do - new_span = Span.child_of(span, name, Utils.next_id(), Utils.now(), opts) - - case new_span do - {:error, errors} -> - {:error, errors} - - new_span -> - put_trace(%{id: trace_id, stack: [new_span], spans: []}) - - Logger.metadata(span_id: new_span.id) - - {:ok, new_span.id} - end - end - - @doc """ - Continues a trace given a name, a trace_id and a span_id - """ - @impl Spandex.Adapters.Adapter - @spec continue_trace(String.t(), term, term, Keyword.t()) :: {:ok, term} | {:error, term} - def continue_trace(name, trace_id, span_id, opts) - when is_integer(trace_id) and is_integer(span_id) do - trace = get_trace(:undefined) - - cond do - trace == :undefined -> - top_span = - opts - |> Keyword.put_new(:name, name) - |> Keyword.put(:trace_id, trace_id) - |> Keyword.put(:start, Utils.now()) - |> Keyword.put(:parent_id, span_id) - |> Keyword.put(:id, Utils.next_id()) - |> Span.new() - - put_trace(%{id: trace_id, stack: [top_span], spans: [], start: Utils.now()}) - {:ok, trace_id} - - trace_id == trace.id -> - start_span(name, opts) - - true -> - {:error, :trace_already_present} - end - end - - @doc """ - Attaches error data to the current span, and marks it as an error. - """ - @impl Spandex.Adapters.Adapter - @spec span_error(Exception.t(), Keyword.t()) :: :ok | {:error, term} - def span_error(exception, opts) do - updates = [error: [exception: exception, stacktrace: System.stacktrace()]] - - update_span(Keyword.put_new(opts, :error, updates)) + def default_sender() do + Spandex.Datadog.ApiServer end @doc """ @@ -376,20 +57,4 @@ defmodule Spandex.Adapters.Datadog do end defp parse_header(_header), do: nil - - @spec get_trace(term) :: term - defp get_trace(default \\ nil) do - Process.get(:spandex_trace, default) - end - - @spec put_trace(term) :: term | nil - defp put_trace(updates) do - Process.put(:spandex_trace, updates) - end - - @spec delete_trace() :: term | nil - defp delete_trace() do - Logger.metadata(trace_id: nil, span_id: nil) - Process.delete(:spandex_trace) - end end diff --git a/lib/datadog/api_server.ex b/lib/datadog/api_server.ex index 70448b9..760b27f 100644 --- a/lib/datadog/api_server.ex +++ b/lib/datadog/api_server.ex @@ -201,7 +201,7 @@ defmodule Spandex.Datadog.ApiServer do span_id: span.id, name: span.name, start: span.start, - duration: (span.completion_time || Spandex.Datadog.Utils.now()) - span.start, + duration: (span.completion_time || Spandex.Adapters.Datadog.now()) - span.start, parent_id: span.parent_id, error: error(span.error), resource: span.resource, @@ -233,11 +233,15 @@ defmodule Spandex.Datadog.ApiServer do defp add_error_data(meta, %{error: error}) do meta - |> Map.put("error.type", error.__struct__) + |> add_error_type(error.exception) |> add_error_message(error.exception) |> add_error_stacktrace(error.stacktrace) end + @spec add_error_type(map, Exception.t() | nil) :: map + defp add_error_type(meta, nil), do: meta + defp add_error_type(meta, exception), do: Map.put(meta, "error.type", exception.__struct__) + @spec add_error_message(map, Exception.t() | nil) :: map defp add_error_message(meta, nil), do: meta @@ -248,7 +252,7 @@ defmodule Spandex.Datadog.ApiServer do defp add_error_stacktrace(meta, nil), do: meta defp add_error_stacktrace(meta, stacktrace), - do: Map.put(meta, "error.msg", Exception.format_stacktrace(stacktrace)) + do: Map.put(meta, "error.stack", Exception.format_stacktrace(stacktrace)) @spec add_http_data(map, Spandex.Span.t()) :: map defp add_http_data(meta, %{http: nil}), do: meta diff --git a/lib/datadog/utils.ex b/lib/datadog/utils.ex deleted file mode 100644 index c350808..0000000 --- a/lib/datadog/utils.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Spandex.Datadog.Utils do - @moduledoc """ - A set of common utils specific (for now) to DataDog APM). - """ - - @max_id 9_223_372_036_854_775_807 - - @doc """ - Returns the current timestamp in nanoseconds. - """ - @spec now() :: non_neg_integer - def now() do - :os.system_time(:nano_seconds) - end - - @doc """ - Generates new span compatible id. - """ - @spec next_id() :: non_neg_integer - def next_id() do - :rand.uniform(@max_id) - end -end diff --git a/lib/span.ex b/lib/span.ex index 4bce15d..ed86781 100644 --- a/lib/span.ex +++ b/lib/span.ex @@ -108,7 +108,7 @@ defmodule Spandex.Span do left = struct_to_keyword(v1) right = struct_to_keyword(v2) - Keyword.merge(left, right) + merge_non_nils(left, right) :tags -> Keyword.merge(v1 || [], v2 || []) @@ -130,6 +130,17 @@ defmodule Spandex.Span do validate_and_merge(span, with_type, schema) end + @spec merge_non_nils(Keyword.t(), Keyword.t()) :: Keyword.t() + defp merge_non_nils(left, right) do + Keyword.merge(left, right, fn _k, v1, v2 -> + if is_nil(v2) do + v1 + else + v2 + end + end) + end + @spec validate_and_merge(t(), Keyword.t(), Optimal.schema()) :: t() | {:error, term} defp validate_and_merge(span, opts, schema) do case Optimal.validate(opts, schema) do diff --git a/lib/spandex.ex b/lib/spandex.ex index 2b71dd0..c45c539 100644 --- a/lib/spandex.ex +++ b/lib/spandex.ex @@ -4,109 +4,273 @@ defmodule Spandex do """ require Logger + alias Spandex.Span + alias Spandex.Trace + + def start_trace(_, :disabled), do: {:error, :disabled} + def start_trace(name, opts) do - adapter = opts[:adapter] + strategy = opts[:strategy] + + if strategy.get_trace(opts[:tracer]) do + _ = Logger.error("Tried to start a trace over top of another trace.") + {:error, :trace_running} + else + adapter = opts[:adapter] + trace_id = adapter.trace_id() - case adapter.start_trace(name, opts) do - {:ok, trace_id} -> - Logger.metadata(trace_id: trace_id) + name + |> span(opts, trace_id, adapter) + |> with_span(fn span -> + Logger.metadata(trace_id: trace_id, span_id: span.id) - adapter.update_span(opts) - {:ok, trace_id} + trace = %Trace{ + spans: [], + stack: [span], + id: trace_id + } - {:error, error} -> - {:error, error} + strategy.put_trace(opts[:tracer], trace) + end) end end + def start_span(_, :disabled), do: {:error, :disabled} + def start_span(name, opts) do + strategy = opts[:strategy] adapter = opts[:adapter] - case adapter.start_span(name, opts) do - {:ok, span_id} -> - Logger.metadata(span_id: span_id) + case strategy.get_trace(opts[:tracer]) do + nil -> + {:error, :no_trace_context} + + %Trace{stack: [current_span | _]} = trace -> + current_span + |> Span.child_of(name, adapter.span_id(), adapter.now(), opts) + |> with_span(fn span -> + strategy.put_trace(opts[:tracer], %{trace | stack: [span | trace.stack]}) + + Logger.metadata(span_id: span.id) + + {:ok, span} + end) - adapter.update_span(opts) - {:ok, span_id} + %Trace{stack: [], id: trace_id} = trace -> + name + |> span(opts, trace_id, adapter) + |> with_span(fn span -> + strategy.put_trace(opts[:tracer], %{trace | stack: [span]}) - {:error, error} -> - {:error, error} + Logger.metadata(span_id: span.id) + + {:ok, span} + end) end end - def update_span(opts) do - adapter = opts[:adapter] + def update_span(opts, top? \\ false) + def update_span(:disabled, _), do: {:error, :disabled} + + def update_span(opts, top?) do + strategy = opts[:strategy] + + case strategy.get_trace(opts[:tracer]) do + nil -> + {:error, :no_trace_context} + + %Trace{stack: stack} = trace -> + index = + if top? do + -1 + else + 0 + end - adapter.update_span(opts) + new_stack = List.update_at(stack, index, &update_or_keep(&1, opts)) + + strategy.put_trace(opts[:tracer], %{trace | stack: new_stack}) + + {:ok, Enum.at(new_stack, index)} + end end + def update_top_span(:disabled), do: {:error, :disabled} + def update_top_span(opts) do - adapter = opts[:adapter] + top? = true + update_span(opts, top?) + end + + def update_all_spans(:disabled), do: {:error, :disabled} + + def update_all_spans(opts) do + strategy = opts[:strategy] + + case strategy.get_trace(opts[:tracer]) do + nil -> + {:error, :no_trace_context} + + %Trace{stack: stack, spans: spans} = trace -> + new_stack = Enum.map(stack, &update_or_keep(&1, opts)) - adapter.update_top_span(opts) + new_spans = Enum.map(spans, &update_or_keep(&1, opts)) + + strategy.put_trace(opts[:tracer], %{trace | stack: new_stack, spans: new_spans}) + end end - # All of these need to honor `disabled?: true` + def finish_trace(:disabled), do: {:error, :disabled} + def finish_trace(opts) do - wrap_adapter(opts, fn adapter -> - adapter.finish_trace(opts) - end) + strategy = opts[:strategy] + adapter = opts[:adapter] + + case strategy.get_trace(opts[:tracer]) do + nil -> + {:error, :no_trace_context} + + %Trace{spans: spans, stack: stack} -> + unfinished_spans = Enum.map(stack, &update_or_keep(&1, completion_time: adapter.now())) + + sender = opts[:sender] || adapter.default_sender() + + spans + |> Kernel.++(unfinished_spans) + |> sender.send_spans() + + strategy.delete_trace(opts[:tracer]) + end end + def finish_span(:disabled), do: {:error, :disabled} + def finish_span(opts) do - wrap_adapter(opts, fn adapter -> - adapter.finish_span(opts) - end) - end + strategy = opts[:strategy] + adapter = opts[:adapter] - def span_error(error, opts) do - wrap_adapter(opts, fn adapter -> - adapter.span_error(error, opts) - end) - end + case strategy.get_trace(opts[:tracer]) do + nil -> + {:error, :no_trace_context} - def continue_trace(name, trace_id, span_id, opts) do - wrap_adapter(opts, fn adapter -> - adapter.continue_trace(name, trace_id, span_id, opts) - end) + %Trace{stack: []} -> + {:error, :no_span_context} + + %Trace{stack: [span | tail], spans: spans} = trace -> + finished_span = update_or_keep(span, completion_time: adapter.now()) + + strategy.put_trace(opts[:tracer], %{trace | stack: tail, spans: [finished_span | spans]}) + {:ok, finished_span} + end end - def continue_trace_from_span(name, span, opts) do - wrap_adapter(opts, fn adapter -> - adapter.continue_trace_from_span(name, span, opts) - end) + def span_error(_error, _stacktrace, :disabled), do: {:error, :disabled} + + def span_error(exception, stacktrace, opts) do + updates = [exception: exception, stacktrace: stacktrace] + + update_span(Keyword.put_new(opts, :error, updates)) end + def current_trace_id(:disabled), do: nil + def current_trace_id(opts) do - wrap_adapter(opts, nil, fn adapter -> - adapter.current_trace_id(opts) - end) + strategy = opts[:strategy] + + case strategy.get_trace(opts[:tracer]) do + nil -> + nil + + %Trace{id: id} -> + id + end end + def current_span_id(:disabled), do: nil + def current_span_id(opts) do - wrap_adapter(opts, nil, fn adapter -> - adapter.current_span_id(opts) - end) + case current_span(opts) do + nil -> nil + span -> span.id + end end + def current_span(:disabled), do: nil + def current_span(opts) do - wrap_adapter(opts, nil, fn adapter -> - adapter.current_span(opts) - end) + strategy = opts[:strategy] + + case strategy.get_trace(opts[:tracer]) do + nil -> nil + %Trace{stack: []} -> nil + %Trace{stack: [span | _]} -> span + end end + def continue_trace(_, _, _, :disabled), do: {:error, :disabled} + + def continue_trace(name, trace_id, span_id, opts) do + strategy = opts[:strategy] + adapter = opts[:adapter] + + case strategy.get_trace(opts[:tracer]) do + nil -> + opts_with_parent = Keyword.put(opts, :parent_id, span_id) + top_span = span(name, opts_with_parent, trace_id, adapter) + + strategy.put_trace(opts[:tracer], %Trace{id: trace_id, stack: [top_span], spans: []}) + + _ -> + {:error, :trace_already_present} + end + end + + def continue_trace_from_span(_name, _span, :disabled), do: {:error, :disabled} + + def continue_trace_from_span(name, span, opts) do + strategy = opts[:strategy] + adapter = opts[:adapter] + + case strategy.get_trace(opts[:tracer]) do + nil -> + span + |> Span.child_of(name, adapter.span_id(), adapter.now(), opts) + |> with_span(fn span -> + strategy.put_trace(opts[:tracer], %Trace{ + id: adapter.trace_id(), + stack: [span], + spans: [] + }) + end) + + _ -> + {:error, :trace_already_present} + end + end + + def distributed_context(_, :disabled), do: {:error, :disabled} + def distributed_context(conn, opts) do - wrap_adapter(opts, {:error, :disabled}, fn adapter -> - adapter.distributed_context(conn, opts) - end) + adapter = opts[:adapter] + adapter.distributed_context(conn, opts) end - defp wrap_adapter(opts, ret \\ {:ok, :disabled}, fun) do - if opts[:disabled?] == true do - ret - else - adapter = opts[:adapter] - fun.(adapter) + defp span(name, opts, trace_id, adapter) do + opts + |> Keyword.put_new(:name, name) + |> Keyword.put(:trace_id, trace_id) + |> Keyword.put(:start, adapter.now()) + |> Keyword.put(:id, adapter.span_id()) + |> Span.new() + end + + defp update_or_keep(span, opts) do + case Span.update(span, opts) do + {:error, _} -> span + span -> span end end + + def with_span({:error, errors}, _fun), do: {:error, errors} + def with_span(span, fun), do: fun.(span) end diff --git a/lib/strategy.ex b/lib/strategy.ex new file mode 100644 index 0000000..aa68957 --- /dev/null +++ b/lib/strategy.ex @@ -0,0 +1,12 @@ +defmodule Spandex.Strategy do + @moduledoc """ + The behaviour for a storage strategy for storing an ongoing trace. + """ + @type tracer :: module + + alias Spandex.Trace + + @callback get_trace(tracer) :: Trace.t() | nil + @callback put_trace(tracer, Trace.t()) :: {:ok, Trace.t()} | {:error, term} + @callback delete_trace(tracer) :: :ok | {:error, term} +end diff --git a/lib/strategy/pdict.ex b/lib/strategy/pdict.ex new file mode 100644 index 0000000..49a7938 --- /dev/null +++ b/lib/strategy/pdict.ex @@ -0,0 +1,27 @@ +defmodule Spandex.Strategy.Pdict do + @moduledoc """ + This stores traces in the local process dictionary, scoped by the + tracer running the trace, such that you could have multiple traces + going at one time by using a different tracer. + """ + @behaviour Spandex.Strategy + + @impl Spandex.Strategy + def get_trace(tracer) do + Process.get({:spandex_trace, tracer}) + end + + @impl Spandex.Strategy + def put_trace(tracer, trace) do + Process.put({:spandex_trace, tracer}, trace) + + {:ok, trace} + end + + @impl Spandex.Strategy + def delete_trace(tracer) do + Process.delete({:spandex_trace, tracer}) + + :ok + end +end diff --git a/lib/trace.ex b/lib/trace.ex index d8f04ef..0bb114a 100644 --- a/lib/trace.ex +++ b/lib/trace.ex @@ -11,7 +11,6 @@ defmodule Spandex.Trace do @type t :: %__MODULE__{ stack: [Spandex.Span.t()], spans: [Spandex.Span.t()], - id: term, - start: term + id: term } end diff --git a/lib/tracer.ex b/lib/tracer.ex index e27c764..26bcae5 100644 --- a/lib/tracer.ex +++ b/lib/tracer.ex @@ -10,24 +10,29 @@ defmodule Spandex.Tracer do end ``` """ - @type tagged_tuple :: {:ok, term} | {:error, term} + + alias Spandex.Trace + alias Spandex.Span + + @type tagged_tuple(arg) :: {:ok, arg} | {:error, term} @type span_name() :: String.t() @type opts :: Keyword.t() @callback configure(opts) :: :ok - @callback start_trace(span_name, opts) :: tagged_tuple - @callback start_span(span_name, opts) :: tagged_tuple - @callback update_span(opts) :: tagged_tuple - @callback update_top_span(opts) :: tagged_tuple - @callback finish_trace(opts) :: tagged_tuple - @callback finish_span(opts) :: tagged_tuple - @callback span_error(error :: Exception.t(), opts) :: tagged_tuple - @callback continue_trace(span_name, trace_id :: term, span_id :: term, opts) :: tagged_tuple - @callback continue_trace_from_span(span_name, span :: term, opts) :: tagged_tuple - @callback current_trace_id(opts) :: tagged_tuple - @callback current_span_id(opts) :: tagged_tuple - @callback current_span(opts) :: tagged_tuple - @callback distributed_context(Plug.Conn.t(), opts) :: tagged_tuple + @callback start_trace(span_name, opts) :: tagged_tuple(Trace.t()) + @callback start_span(span_name, opts) :: tagged_tuple(Span.t()) + @callback update_span(opts) :: tagged_tuple(Span.t()) + @callback update_top_span(opts) :: tagged_tuple(Span.t()) + @callback finish_trace(opts) :: tagged_tuple(Trace.t()) + @callback finish_span(opts) :: tagged_tuple(Span.t()) + @callback span_error(error :: Exception.t(), opts) :: tagged_tuple(Span.t()) + @callback continue_trace(span_name, trace_id :: term, span_id :: term, opts) :: + tagged_tuple(Trace.t()) + @callback continue_trace_from_span(span_name, span :: term, opts) :: tagged_tuple(Trace.t()) + @callback current_trace_id(opts) :: nil | term + @callback current_span_id(opts) :: nil | term + @callback current_span(opts) :: nil | Span.t() + @callback distributed_context(Plug.Conn.t(), opts) :: tagged_tuple(map) @macrocallback span(span_name, opts, do: Macro.t()) :: Macro.t() @macrocallback trace(span_name, opts, do: Macro.t()) :: Macro.t() @@ -38,26 +43,31 @@ defmodule Spandex.Tracer do disabled?: :boolean, env: :string, services: {:keyword, :atom}, - sender: :atom + strategy: :atom, + sender: :atom, + tracer: :atom ], required: [:adapter, :service], defaults: [ disabled?: false, env: "unknown", services: [], - sender: Spandex.Datadog.ApiServer + strategy: Spandex.Strategy.Pdict ], describe: [ - adapter: "The third party adapter to use.", + adapter: "The third party adapter to use", + tracer: "Don't set manually. This option is passed automatically.", + sender: + "Once a trace is complete, it is sent using this module. Defaults to the `default_sender/0` of the selected adapter", service: - "The default service name to use for spans declared without a service.", - disabled?: "Allows for wholesale disabling a tracer.", + "The default service name to use for spans declared without a service", + disabled?: "Allows for wholesale disabling a tracer", env: "A name used to identify the environment name, e.g `prod` or `development`", services: "A mapping of service name to the default span types. For instance datadog knows about `:db`, `:cache` and `:web`", - sender: - "Will be deprecated soon, but this is a module that defines a `send_spans/1` function." + strategy: + "The storage and tracing strategy. Currently only supports local process dictionary." ] ) @@ -93,12 +103,19 @@ defmodule Spandex.Tracer do @doc """ Use to create and configure a tracer. """ + @impl Spandex.Tracer @spec configure(Tracer.opts()) :: :ok def configure(opts) do - config = config(opts, @otp_app) - Application.put_env(@otp_app, __MODULE__, config) + case config(opts, @otp_app) do + :disabled -> + :ok + + config -> + Application.put_env(@otp_app, __MODULE__, config) + end end + @impl Spandex.Tracer defmacro trace(name, opts \\ [], do: body) do quote do opts = unquote(opts) @@ -113,7 +130,7 @@ defmodule Spandex.Tracer do rescue exception -> stacktrace = System.stacktrace() - _ = unquote(__MODULE__).span_error(exception, opts) + _ = unquote(__MODULE__).span_error(exception, stacktrace, opts) reraise exception, stacktrace after unquote(__MODULE__).finish_trace() @@ -121,6 +138,7 @@ defmodule Spandex.Tracer do end end + @impl Spandex.Tracer defmacro span(name, opts \\ [], do: body) do quote do opts = unquote(opts) @@ -134,7 +152,7 @@ defmodule Spandex.Tracer do rescue exception -> stacktrace = System.stacktrace() - _ = unquote(__MODULE__).span_error(exception, opts) + _ = unquote(__MODULE__).span_error(exception, stacktrace, opts) reraise exception, stacktrace after unquote(__MODULE__).finish_span() @@ -142,83 +160,89 @@ defmodule Spandex.Tracer do end end - @spec start_trace(Tracer.span_name(), Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def start_trace(name, opts \\ []) do Spandex.start_trace(name, config(opts, @otp_app)) end - @spec start_span(Tracer.span_name(), Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def start_span(name, opts \\ []) do Spandex.start_span(name, config(opts, @otp_app)) end - @spec update_span(Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def update_span(opts) do Spandex.update_span(config(opts, @otp_app)) end - @spec update_top_span(Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def update_top_span(opts) do Spandex.update_top_span(config(opts, @otp_app)) end - @spec finish_trace(Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def finish_trace(opts \\ []) do opts |> config(@otp_app) |> Spandex.finish_trace() end - @spec finish_span(Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def finish_span(opts \\ []) do opts |> config(@otp_app) |> Spandex.finish_span() end - @spec span_error(error :: Exception.t(), Tracer.opts()) :: Tracer.tagged_tuple() - def span_error(error, opts \\ []) do - Spandex.span_error(error, config(opts, @otp_app)) + @impl Spandex.Tracer + def span_error(error, stacktrace, opts \\ []) do + Spandex.span_error(error, stacktrace, config(opts, @otp_app)) end - @spec continue_trace(Tracer.span_name(), trace_id :: term, span_id :: term, Tracer.opts()) :: - Tracer.tagged_tuple() + @impl Spandex.Tracer def continue_trace(span_name, trace_id, span_id, opts \\ []) do Spandex.continue_trace(span_name, trace_id, span_id, config(opts, @otp_app)) end - @spec continue_trace_from_span(Tracer.span_name(), span :: term, Tracer.opts()) :: - Tracer.tagged_tuple() + @impl Spandex.Tracer def continue_trace_from_span(span_name, span, opts \\ []) do Spandex.continue_trace_from_span(span_name, span, config(opts, @otp_app)) end - @spec current_trace_id(Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def current_trace_id(opts \\ []) do Spandex.current_trace_id(config(opts, @otp_app)) end - @spec current_span_id(Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def current_span_id(opts \\ []) do Spandex.current_span_id(config(opts, @otp_app)) end - @spec current_span(Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def current_span(opts \\ []) do Spandex.current_span(config(opts, @otp_app)) end - @spec distributed_context(conn :: Plug.Conn.t(), Tracer.opts()) :: Tracer.tagged_tuple() + @impl Spandex.Tracer def distributed_context(conn, opts \\ []) do Spandex.distributed_context(conn, config(opts, @otp_app)) end - def config(opts, otp_app) do - otp_app - |> Application.get_env(__MODULE__) - |> Kernel.||([]) - |> Keyword.merge(opts || []) - |> Optimal.validate!(@opts) + defp config(opts, otp_app) do + config = + otp_app + |> Application.get_env(__MODULE__) + |> Kernel.||([]) + |> Keyword.merge(opts || []) + |> Optimal.validate!(@opts) + |> Keyword.put(:tracer, __MODULE__) + + if config[:disabled?] do + :disabled + else + config + end end end end diff --git a/mix.exs b/mix.exs index e980569..c2a3226 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule Spandex.Mixfile do [ app: :spandex, version: "1.5.0", - elixir: "~> 1.5", + elixir: "~> 1.3", elixirc_paths: elixirc_paths(Mix.env()), build_embedded: Mix.env() == :prod, start_permanent: Mix.env() == :prod, diff --git a/test/plug/add_context_test.exs b/test/plug/add_context_test.exs index 89dffb1..ca046aa 100644 --- a/test/plug/add_context_test.exs +++ b/test/plug/add_context_test.exs @@ -6,12 +6,12 @@ defmodule Spandex.Plug.AddContextTest do alias Spandex.Test.Support.Tracer setup do - {:ok, trace_id} = Tracer.start_trace("request") + {:ok, trace} = Tracer.start_trace("request") { :ok, [ - trace_id: trace_id, + trace_id: trace.id, conn: Plug.Adapters.Test.Conn.conn(%Plug.Conn{}, :get, "/dashboard", nil) ] } @@ -30,9 +30,6 @@ defmodule Spandex.Plug.AddContextTest do :ok = Tracer.finish_trace() - refute Keyword.has_key?(Logger.metadata(), :trace_id) - refute Keyword.has_key?(Logger.metadata(), :span_id) - %{resource: resource, meta: meta} = Spandex.Test.Util.find_span("request") assert is_nil(Map.get(meta, "http.url")) @@ -51,10 +48,10 @@ defmodule Spandex.Plug.AddContextTest do tracer_opts: [] ) - {:ok, expected_span_id} = Tracer.start_span("foobar") + {:ok, expected_span} = Tracer.start_span("foobar") assert Keyword.fetch!(Logger.metadata(), :trace_id) == tid - assert Keyword.fetch!(Logger.metadata(), :span_id) == expected_span_id + assert Keyword.fetch!(Logger.metadata(), :span_id) == expected_span.id :ok = Tracer.finish_trace() diff --git a/test/plug/end_trace_test.exs b/test/plug/end_trace_test.exs index 0a3802f..b81c9d1 100644 --- a/test/plug/end_trace_test.exs +++ b/test/plug/end_trace_test.exs @@ -6,12 +6,12 @@ defmodule Spandex.Plug.EndTraceTest do alias Spandex.Test.Support.Tracer setup do - {:ok, trace_id} = Tracer.start_trace("request") + {:ok, trace} = Tracer.start_trace("request") { :ok, [ - trace_id: trace_id, + trace_id: trace.id, conn: Plug.Adapters.Test.Conn.conn(%Plug.Conn{}, :get, "/dashboard", nil) ] }