packaging for obs.osmocom.org, based on https://salsa.debian.org/erlang-team/packages/rebar3
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
rebar3/src/rebar_compiler.erl

419 lines
17 KiB

-module(rebar_compiler).
-export([analyze_all/2,
analyze_all_extras/2,
compile_analyzed/3,
compile_all/2,
clean/2,
needs_compile/3,
ok_tuple/2,
error_tuple/4,
maybe_report/1,
format_error_source/2,
report/1]).
-include("rebar.hrl").
-type extension() :: string().
-type out_mappings() :: [{extension(), file:filename()}].
-callback context(rebar_app_info:t()) -> #{src_dirs => [DirName :: file:filename()], % mandatory
include_dirs => [DirName :: file:filename()], % mandatory
src_ext => extension(), % mandatory
out_mappings => out_mappings(), % mandatory
dependencies_opts => term()}. % optional
-callback needed_files(digraph:graph(), [file:filename()], out_mappings(),
rebar_app_info:t()) ->
{{[file:filename()], term()}, % ErlFirstFiles (erl_opts global priority)
{[file:filename()] | % [Sequential]
{[file:filename()], [file:filename()]}, % {Sequential, Parallel}
term()}}.
-callback dependencies(file:filename(), DirName :: file:filename(), [DirName :: file:filename()]) -> [file:filename()].
-callback dependencies(file:filename(), DirName :: file:filename(), [DirName :: file:filename()], term()) -> [file:filename()].
-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
ok | {ok, [string()]} | error | {error, [string()], [string()]} | skipped.
-callback compile_and_track(file:filename(), out_mappings(), rebar_dict(), list()) ->
{ok, [{file:filename(), file:filename(), term()}]} |
{ok, [{file:filename(), file:filename(), term()}], [string()]} |
{error, [string()], [string()]} | error.
-callback clean([file:filename()], rebar_app_info:t()) -> _.
-optional_callbacks([dependencies/4, compile_and_track/4]).
%% @doc analysis by the caller, in order to let an OTP app
%% find and resolve all its dependencies as part of compile_all's new
%% API, which presumes a partial analysis is done ahead of time
-spec analyze_all(DAG, [App, ...]) -> {map(), [App]} when
DAG :: {module(), digraph:graph()},
App :: rebar_app_info:t().
analyze_all({Compiler, G}, Apps) ->
prepare_compiler_env(Compiler, Apps),
%% Analyze apps one by one
%% then cover the include files in the digraph to update them
%% then propagate?
Contexts = gather_contexts(Compiler, Apps),
AppRes = [analyze_app({Compiler, G}, Contexts, AppInfo) || AppInfo <- Apps],
{AppOutPaths, AbsSources} = lists:unzip(AppRes),
SrcExt = maps:get(src_ext, Contexts),
OutExt = maps:get(artifact_exts, Contexts),
rebar_compiler_dag:prune(
G, SrcExt, OutExt, lists:append(AbsSources), lists:append(AppOutPaths)
),
rebar_compiler_dag:populate_deps(G, SrcExt, OutExt),
rebar_compiler_dag:propagate_stamps(G),
[$a, Sep, $b] = filename:join("a", "b"),
AppPaths = [{rebar_app_info:name(AppInfo),
rebar_utils:to_list(rebar_app_info:dir(AppInfo)) ++ [Sep]}
|| AppInfo <- Apps],
AppNames = rebar_compiler_dag:compile_order(G, AppPaths, SrcExt, OutExt),
{Contexts, sort_apps(AppNames, Apps)}.
%% @doc same as analyze_all/2, but over extra_src_apps,
%% which are a big cheat.
-spec analyze_all_extras(DAG, [App, ...]) -> {map(), [App]} when
DAG :: {module(), digraph:graph()},
App :: rebar_app_info:t().
analyze_all_extras(DAG, Apps) ->
case lists:append([annotate_extras(App) || App <- Apps]) of
[] -> {#{}, []};
ExtraApps -> analyze_all(DAG, ExtraApps)
end.
-spec compile_analyzed({module(), digraph:graph()}, rebar_app_info:t(), map()) -> ok.
compile_analyzed({Compiler, G}, AppInfo, Contexts) -> % > 3.13.2
run(G, Compiler, AppInfo, Contexts),
ok.
-spec compile_all([module(), ...], rebar_app_info:t()) -> ok.
compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
%% Support the old-style API by re-declaring a local DAG for the
%% compile steps needed.
lists:foreach(fun(Compiler) ->
OutDir = rebar_app_info:out_dir(AppInfo),
G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []),
{Ctx, _} = analyze_all({Compiler, G}, [AppInfo]),
compile_analyzed({Compiler, G}, AppInfo, Ctx),
rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []),
rebar_compiler_dag:terminate(G)
end, Compilers).
%% @doc remove compiled artifacts from an AppDir.
-spec clean([module()], rebar_app_info:t()) -> 'ok'.
clean(Compilers, AppInfo) ->
lists:foreach(fun(CompilerMod) ->
clean_(CompilerMod, AppInfo, undefined),
Extras = annotate_extras(AppInfo),
[clean_(CompilerMod, ExtraApp, "extra") || ExtraApp <- Extras]
end, Compilers).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% COMPILER UTIL EXPORTS %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% These functions are here for the ultimate goal of getting rid of
%% rebar_base_compiler. This can't be done because of existing plugins.
-spec needs_compile(file:name_all(), extension(), [{extension(), DirName :: file:filename()}]) -> boolean().
needs_compile(Source, OutExt, Mappings) ->
Ext = filename:extension(Source),
BaseName = filename:basename(Source, Ext),
{_, OutDir} = lists:keyfind(OutExt, 1, Mappings),
Target = filename:join(OutDir, BaseName++OutExt),
filelib:last_modified(Source) > filelib:last_modified(Target).
ok_tuple(Source, Ws) ->
rebar_base_compiler:ok_tuple(Source, Ws).
error_tuple(Source, Es, Ws, Opts) ->
rebar_base_compiler:error_tuple(Source, Es, Ws, Opts).
maybe_report(Reportable) ->
rebar_base_compiler:maybe_report(Reportable).
format_error_source(Path, Opts) ->
rebar_base_compiler:format_error_source(Path, Opts).
report(Messages) ->
rebar_base_compiler:report(Messages).
%%%%%%%%%%%%%%%
%%% PRIVATE %%%
%%%%%%%%%%%%%%%
gather_contexts(Compiler, Apps) ->
Default = default_ctx(),
Contexts = [{rebar_app_info:name(AppInfo),
maps:merge(Default, Compiler:context(AppInfo))}
|| AppInfo <- Apps],
ContextMap = maps:from_list(Contexts),
%% only support one extension type at once for now
[{_, #{src_ext := SrcExt}} | _] = Contexts,
%% gather multi-app stuff once to avoid recomputing it
ArtifactExts = lists:usort(
[Ext || {_, #{out_mappings := Mappings}} <- Contexts,
{Ext, _Dir} <- Mappings]
),
InDirs = gather_in_dirs(lists:zip(Apps, [Context || {_, Context} <- Contexts])),
ContextMap#{src_ext => SrcExt,
artifact_exts => ArtifactExts,
in_dirs => InDirs}.
gather_in_dirs(AppCtx) ->
gather_in_dirs(AppCtx, []).
gather_in_dirs([], Paths) ->
lists:usort(Paths);
gather_in_dirs([{AppInfo, Ctx} | Rest], Acc) ->
#{include_dirs := InclDirs,
src_dirs := SrcDirs} = Ctx,
BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
AbsIncl = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
AbsSrc = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
gather_in_dirs(Rest, AbsSrc ++ AbsIncl ++ Acc).
analyze_app({Compiler, G}, Contexts, AppInfo) ->
AppName = rebar_app_info:name(AppInfo),
BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)),
BaseOpts = rebar_app_info:opts(AppInfo),
#{src_dirs := SrcDirs,
src_ext := SrcExt,
out_mappings := [{_OutExt, OutPath}|_], % prune one dir for now (compat mode!)
dependencies_opts := DepOpts} = maps:get(AppName, Contexts),
%% Local resources
ArtifactDir = filename:join([OutDir, OutPath]),
AbsSources = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
%% Multi-app resources
InDirs = maps:get(in_dirs, Contexts),
%% Run the analysis
rebar_compiler_dag:populate_sources(
G, Compiler, InDirs, AbsSources, DepOpts
),
{[{filename:join([BaseDir, SrcDir]), ArtifactDir} || SrcDir <- SrcDirs],
AbsSources}.
sort_apps(Names, Apps) ->
NamedApps = [{rebar_app_info:name(App), App} || App <- Apps],
[App || Name <- Names,
{_, App} <- [lists:keyfind(Name, 1, NamedApps)]].
prepare_compiler_env(Compiler, Apps) ->
RebarLibs = [rebar_utils:to_binary(Atom)
|| {ok, RebarApps} <- [application:get_key(rebar, applications)],
Atom <- RebarApps],
lists:foreach(
fun(AppInfo) ->
EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
%% Make sure that outdir is on the path
ok = rebar_file_utils:ensure_dir(EbinDir),
%% We use code:add_pathz for better caching speed when
%% dealing with overall projects and deps under profiles,
%% but for correctness' sake, we also have to
%% use code:add_patha to go above rebar3's own dependencies
%% when they clash to avoid overtaking the project's
%% path for includes and priv/
case lists:member(rebar_app_info:name(AppInfo), RebarLibs) of
true -> true = code:add_patha(filename:absname(EbinDir));
false -> true = code:add_pathz(filename:absname(EbinDir))
end
end,
Apps
),
%% necessary for erlang:function_exported/3 to work as expected
%% called here for clarity as it's required by both opts_changed/2
%% and erl_compiler_opts_set/0 in needed_files
application:load(compiler),
_ = code:ensure_loaded(compile),
_ = code:ensure_loaded(Compiler),
ok.
run(G, CompilerMod, AppInfo, Contexts) ->
Name = rebar_app_info:name(AppInfo),
#{src_dirs := SrcDirs,
src_ext := SrcExt,
out_mappings := Mappings} = maps:get(Name, Contexts),
BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
BaseOpts = rebar_app_info:opts(AppInfo),
FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
{{FirstFiles, FirstFileOpts},
{RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),
Tracked =
compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod)
++ case RestFiles of
{Sequential, Parallel} -> % parallelizable form
compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod) ++
lists:append(
compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod)
);
_ when is_list(RestFiles) -> % traditional sequential build
compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod)
end,
store_artifacts(G, Tracked).
compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
[];
compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
case erlang:function_exported(CompilerMod, compile_and_track, 4) of
false ->
do_compile(CompilerMod, Source, Outs, Config, Opts),
compile_each(Rest, Opts, Config, Outs, CompilerMod);
true ->
do_compile_and_track(CompilerMod, Source, Outs, Config, Opts)
++ compile_each(Rest, Opts, Config, Outs, CompilerMod)
end.
do_compile(CompilerMod, Source, Outs, Config, Opts) ->
case CompilerMod:compile(Source, Outs, Config, Opts) of
ok ->
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
{ok, Warnings} ->
report(Warnings),
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
skipped ->
?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
Error ->
NewSource = format_error_source(Source, Config),
?ERROR("Compiling ~ts failed", [NewSource]),
maybe_report(Error),
?DEBUG("Compilation failed: ~p", [Error]),
?ABORT
end.
do_compile_and_track(CompilerMod, Source, Outs, Config, Opts) ->
case CompilerMod:compile_and_track(Source, Outs, Config, Opts) of
{ok, Tracked} ->
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
Tracked;
{ok, Tracked, Warnings} ->
report(Warnings),
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
Tracked;
skipped ->
?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
[];
Error ->
NewSource = format_error_source(Source, Config),
?ERROR("Compiling ~ts failed", [NewSource]),
maybe_report(Error),
?DEBUG("Compilation failed: ~p", [Error]),
?ABORT
end.
store_artifacts(_G, []) ->
ok;
store_artifacts(G, [{Source, Target, Meta}|Rest]) ->
%% Assume the source exists since it was tracked to be compiled
rebar_compiler_dag:store_artifact(G, Source, Target, Meta),
store_artifacts(G, Rest).
compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) ->
Tracking = erlang:function_exported(CompilerMod, compile_and_track, 4),
rebar_parallel:queue(
Targets,
fun compile_worker/2, [Opts, BaseOpts, Mappings, CompilerMod],
fun compile_handler/2, [BaseOpts, Tracking]
).
compile_worker(Source, [Opts, Config, Outs, CompilerMod]) ->
Result = case erlang:function_exported(CompilerMod, compile_and_track, 4) of
false ->
CompilerMod:compile(Source, Outs, Config, Opts);
true ->
CompilerMod:compile_and_track(Source, Outs, Config, Opts)
end,
%% Bundle the source to allow proper reporting in the handler:
{Result, Source}.
compile_handler({ok, Source}, _Args) ->
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
ok;
compile_handler({{ok, Tracked}, Source}, [_, Tracking]) when Tracking ->
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
{ok, Tracked};
compile_handler({{ok, Warnings}, Source}, _Args) ->
report(Warnings),
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
ok;
compile_handler({{ok, Tracked, Warnings}, Source}, [_, Tracking]) when Tracking ->
report(Warnings),
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
{ok, Tracked};
compile_handler({skipped, Source}, _Args) ->
?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), filename:basename(Source)]),
ok;
compile_handler({Error, Source}, [Config | _Rest]) ->
NewSource = format_error_source(Source, Config),
?ERROR("Compiling ~ts failed", [NewSource]),
maybe_report(Error),
?ABORT.
clean_(CompilerMod, AppInfo, _Label) ->
#{src_dirs := SrcDirs,
src_ext := SrcExt} = CompilerMod:context(AppInfo),
BaseDir = rebar_app_info:dir(AppInfo),
Opts = rebar_app_info:opts(AppInfo),
FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts),
CompilerMod:clean(FoundFiles, AppInfo),
ok.
annotate_extras(AppInfo) ->
AppOpts = rebar_app_info:opts(AppInfo),
ExtraDirs = rebar_dir:extra_src_dirs(AppOpts, []),
OldSrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]),
%% Re-annotate the directories with non-default options if it is the
%% case; otherwise, later down the line, the options get dropped with
%% profiles. All of this must be done with the rebar_dir functionality
%% which properly tracks and handles the various legacy formats for
%% recursion setting (erl_opts vs. dir options and profiles)
ExtraDirsOpts = [case rebar_dir:recursive(AppOpts, Dir) of
false -> {Dir, [{recursive, false}]};
true -> Dir
end || Dir <- ExtraDirs],
OldSrcDirsOpts = [case rebar_dir:recursive(AppOpts, Dir) of
false -> {Dir, [{recursive, false}]};
true -> Dir
end || Dir <- OldSrcDirs],
AppDir = rebar_app_info:dir(AppInfo),
lists:map(fun({DirOpt, Dir}) ->
EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
%% need a unique name to prevent lookup issues that clobber entries
AppName = unicode:characters_to_binary(
[rebar_app_info:name(AppInfo), "_", Dir]
),
AppInfo0 = rebar_app_info:name(AppInfo, AppName),
AppInfo1 = rebar_app_info:ebin_dir(AppInfo0, EbinDir),
AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [DirOpt]),
AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, OldSrcDirsOpts),
add_to_includes( % give access to .hrl in app's src/
AppInfo3,
[filename:join([AppDir, D]) || D <- OldSrcDirs]
)
end,
[T || T = {_DirOpt, ExtraDir} <- lists:zip(ExtraDirsOpts, ExtraDirs),
filelib:is_dir(filename:join(AppDir, ExtraDir))]
).
find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$],
lists:flatmap(fun(SrcDir) ->
Recursive = rebar_dir:recursive(Opts, SrcDir),
rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive)
end, SrcDirs).
add_to_includes(AppInfo, Dirs) ->
Opts = rebar_app_info:opts(AppInfo),
List = rebar_opts:get(Opts, erl_opts, []),
NewErlOpts = [{i, Dir} || Dir <- Dirs] ++ List,
NewOpts = rebar_opts:set(Opts, erl_opts, NewErlOpts),
rebar_app_info:opts(AppInfo, NewOpts).
default_ctx() ->
#{dependencies_opts => []}.