420 lines
17 KiB
Erlang
420 lines
17 KiB
Erlang
-module(rebar_compiler).
|
|
|
|
-export([analyze_all/2,
|
|
analyze_all_extras/2,
|
|
compile_analyzed/3,
|
|
compile_all/2,
|
|
clean/2,
|
|
|
|
needs_compile/3,
|
|
ok_tuple/2,
|
|
error_tuple/4,
|
|
maybe_report/1,
|
|
format_error_source/2,
|
|
report/1]).
|
|
|
|
-include("rebar.hrl").
|
|
|
|
-type extension() :: string().
|
|
-type out_mappings() :: [{extension(), file:filename()}].
|
|
|
|
-callback context(rebar_app_info:t()) -> #{src_dirs => [DirName :: file:filename()], % mandatory
|
|
include_dirs => [DirName :: file:filename()], % mandatory
|
|
src_ext => extension(), % mandatory
|
|
out_mappings => out_mappings(), % mandatory
|
|
dependencies_opts => term()}. % optional
|
|
-callback needed_files(digraph:graph(), [file:filename()], out_mappings(),
|
|
rebar_app_info:t()) ->
|
|
{{[file:filename()], term()}, % ErlFirstFiles (erl_opts global priority)
|
|
{[file:filename()] | % [Sequential]
|
|
{[file:filename()], [file:filename()]}, % {Sequential, Parallel}
|
|
term()}}.
|
|
-callback dependencies(file:filename(), DirName :: file:filename(), [DirName :: file:filename()]) -> [file:filename()].
|
|
-callback dependencies(file:filename(), DirName :: file:filename(), [DirName :: file:filename()], term()) -> [file:filename()].
|
|
-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
|
|
ok | {ok, [string()]} | error | {error, [string()], [string()]} | skipped.
|
|
-callback compile_and_track(file:filename(), out_mappings(), rebar_dict(), list()) ->
|
|
{ok, [{file:filename(), file:filename(), term()}]} |
|
|
{ok, [{file:filename(), file:filename(), term()}], [string()]} |
|
|
{error, [string()], [string()]} | error.
|
|
-callback clean([file:filename()], rebar_app_info:t()) -> _.
|
|
|
|
-optional_callbacks([dependencies/4, compile_and_track/4]).
|
|
|
|
%% @doc analysis by the caller, in order to let an OTP app
|
|
%% find and resolve all its dependencies as part of compile_all's new
|
|
%% API, which presumes a partial analysis is done ahead of time
|
|
-spec analyze_all(DAG, [App, ...]) -> {map(), [App]} when
|
|
DAG :: {module(), digraph:graph()},
|
|
App :: rebar_app_info:t().
|
|
analyze_all({Compiler, G}, Apps) ->
|
|
prepare_compiler_env(Compiler, Apps),
|
|
%% Analyze apps one by one
|
|
%% then cover the include files in the digraph to update them
|
|
%% then propagate?
|
|
Contexts = gather_contexts(Compiler, Apps),
|
|
AppRes = [analyze_app({Compiler, G}, Contexts, AppInfo) || AppInfo <- Apps],
|
|
{AppOutPaths, AbsSources} = lists:unzip(AppRes),
|
|
SrcExt = maps:get(src_ext, Contexts),
|
|
OutExt = maps:get(artifact_exts, Contexts),
|
|
|
|
rebar_compiler_dag:prune(
|
|
G, SrcExt, OutExt, lists:append(AbsSources), lists:append(AppOutPaths)
|
|
),
|
|
rebar_compiler_dag:populate_deps(G, SrcExt, OutExt),
|
|
rebar_compiler_dag:propagate_stamps(G),
|
|
|
|
[$a, Sep, $b] = filename:join("a", "b"),
|
|
AppPaths = [{rebar_app_info:name(AppInfo),
|
|
rebar_utils:to_list(rebar_app_info:dir(AppInfo)) ++ [Sep]}
|
|
|| AppInfo <- Apps],
|
|
AppNames = rebar_compiler_dag:compile_order(G, AppPaths, SrcExt, OutExt),
|
|
{Contexts, sort_apps(AppNames, Apps)}.
|
|
|
|
%% @doc same as analyze_all/2, but over extra_src_apps,
|
|
%% which are a big cheat.
|
|
-spec analyze_all_extras(DAG, [App, ...]) -> {map(), [App]} when
|
|
DAG :: {module(), digraph:graph()},
|
|
App :: rebar_app_info:t().
|
|
analyze_all_extras(DAG, Apps) ->
|
|
case lists:append([annotate_extras(App) || App <- Apps]) of
|
|
[] -> {#{}, []};
|
|
ExtraApps -> analyze_all(DAG, ExtraApps)
|
|
end.
|
|
|
|
-spec compile_analyzed({module(), digraph:graph()}, rebar_app_info:t(), map()) -> ok.
|
|
compile_analyzed({Compiler, G}, AppInfo, Contexts) -> % > 3.13.2
|
|
run(G, Compiler, AppInfo, Contexts),
|
|
ok.
|
|
|
|
-spec compile_all([module(), ...], rebar_app_info:t()) -> ok.
|
|
compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
|
|
%% Support the old-style API by re-declaring a local DAG for the
|
|
%% compile steps needed.
|
|
lists:foreach(fun(Compiler) ->
|
|
OutDir = rebar_app_info:out_dir(AppInfo),
|
|
G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []),
|
|
{Ctx, _} = analyze_all({Compiler, G}, [AppInfo]),
|
|
compile_analyzed({Compiler, G}, AppInfo, Ctx),
|
|
rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []),
|
|
rebar_compiler_dag:terminate(G)
|
|
end, Compilers).
|
|
|
|
%% @doc remove compiled artifacts from an AppDir.
|
|
-spec clean([module()], rebar_app_info:t()) -> 'ok'.
|
|
clean(Compilers, AppInfo) ->
|
|
lists:foreach(fun(CompilerMod) ->
|
|
clean_(CompilerMod, AppInfo, undefined),
|
|
Extras = annotate_extras(AppInfo),
|
|
[clean_(CompilerMod, ExtraApp, "extra") || ExtraApp <- Extras]
|
|
end, Compilers).
|
|
|
|
|
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
|
%%% COMPILER UTIL EXPORTS %%%
|
|
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
|
%% These functions are here for the ultimate goal of getting rid of
|
|
%% rebar_base_compiler. This can't be done because of existing plugins.
|
|
|
|
-spec needs_compile(file:name_all(), extension(), [{extension(), DirName :: file:filename()}]) -> boolean().
|
|
needs_compile(Source, OutExt, Mappings) ->
|
|
Ext = filename:extension(Source),
|
|
BaseName = filename:basename(Source, Ext),
|
|
{_, OutDir} = lists:keyfind(OutExt, 1, Mappings),
|
|
Target = filename:join(OutDir, BaseName++OutExt),
|
|
filelib:last_modified(Source) > filelib:last_modified(Target).
|
|
|
|
ok_tuple(Source, Ws) ->
|
|
rebar_base_compiler:ok_tuple(Source, Ws).
|
|
|
|
error_tuple(Source, Es, Ws, Opts) ->
|
|
rebar_base_compiler:error_tuple(Source, Es, Ws, Opts).
|
|
|
|
maybe_report(Reportable) ->
|
|
rebar_base_compiler:maybe_report(Reportable).
|
|
|
|
format_error_source(Path, Opts) ->
|
|
rebar_base_compiler:format_error_source(Path, Opts).
|
|
|
|
report(Messages) ->
|
|
rebar_base_compiler:report(Messages).
|
|
|
|
%%%%%%%%%%%%%%%
|
|
%%% PRIVATE %%%
|
|
%%%%%%%%%%%%%%%
|
|
|
|
gather_contexts(Compiler, Apps) ->
|
|
Default = default_ctx(),
|
|
Contexts = [{rebar_app_info:name(AppInfo),
|
|
maps:merge(Default, Compiler:context(AppInfo))}
|
|
|| AppInfo <- Apps],
|
|
ContextMap = maps:from_list(Contexts),
|
|
%% only support one extension type at once for now
|
|
[{_, #{src_ext := SrcExt}} | _] = Contexts,
|
|
%% gather multi-app stuff once to avoid recomputing it
|
|
ArtifactExts = lists:usort(
|
|
[Ext || {_, #{out_mappings := Mappings}} <- Contexts,
|
|
{Ext, _Dir} <- Mappings]
|
|
),
|
|
InDirs = gather_in_dirs(lists:zip(Apps, [Context || {_, Context} <- Contexts])),
|
|
ContextMap#{src_ext => SrcExt,
|
|
artifact_exts => ArtifactExts,
|
|
in_dirs => InDirs}.
|
|
|
|
gather_in_dirs(AppCtx) ->
|
|
gather_in_dirs(AppCtx, []).
|
|
|
|
gather_in_dirs([], Paths) ->
|
|
lists:usort(Paths);
|
|
gather_in_dirs([{AppInfo, Ctx} | Rest], Acc) ->
|
|
#{include_dirs := InclDirs,
|
|
src_dirs := SrcDirs} = Ctx,
|
|
BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
|
|
AbsIncl = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
|
|
AbsSrc = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
|
|
gather_in_dirs(Rest, AbsSrc ++ AbsIncl ++ Acc).
|
|
|
|
analyze_app({Compiler, G}, Contexts, AppInfo) ->
|
|
AppName = rebar_app_info:name(AppInfo),
|
|
BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
|
|
OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)),
|
|
BaseOpts = rebar_app_info:opts(AppInfo),
|
|
#{src_dirs := SrcDirs,
|
|
src_ext := SrcExt,
|
|
out_mappings := [{_OutExt, OutPath}|_], % prune one dir for now (compat mode!)
|
|
dependencies_opts := DepOpts} = maps:get(AppName, Contexts),
|
|
%% Local resources
|
|
ArtifactDir = filename:join([OutDir, OutPath]),
|
|
AbsSources = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
|
|
%% Multi-app resources
|
|
InDirs = maps:get(in_dirs, Contexts),
|
|
%% Run the analysis
|
|
rebar_compiler_dag:populate_sources(
|
|
G, Compiler, InDirs, AbsSources, DepOpts
|
|
),
|
|
{[{filename:join([BaseDir, SrcDir]), ArtifactDir} || SrcDir <- SrcDirs],
|
|
AbsSources}.
|
|
|
|
sort_apps(Names, Apps) ->
|
|
NamedApps = [{rebar_app_info:name(App), App} || App <- Apps],
|
|
[App || Name <- Names,
|
|
{_, App} <- [lists:keyfind(Name, 1, NamedApps)]].
|
|
|
|
prepare_compiler_env(Compiler, Apps) ->
|
|
RebarLibs = [rebar_utils:to_binary(Atom)
|
|
|| {ok, RebarApps} <- [application:get_key(rebar, applications)],
|
|
Atom <- RebarApps],
|
|
lists:foreach(
|
|
fun(AppInfo) ->
|
|
EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
|
|
%% Make sure that outdir is on the path
|
|
ok = rebar_file_utils:ensure_dir(EbinDir),
|
|
%% We use code:add_pathz for better caching speed when
|
|
%% dealing with overall projects and deps under profiles,
|
|
%% but for correctness' sake, we also have to
|
|
%% use code:add_patha to go above rebar3's own dependencies
|
|
%% when they clash to avoid overtaking the project's
|
|
%% path for includes and priv/
|
|
case lists:member(rebar_app_info:name(AppInfo), RebarLibs) of
|
|
true -> true = code:add_patha(filename:absname(EbinDir));
|
|
false -> true = code:add_pathz(filename:absname(EbinDir))
|
|
end
|
|
end,
|
|
Apps
|
|
),
|
|
%% necessary for erlang:function_exported/3 to work as expected
|
|
%% called here for clarity as it's required by both opts_changed/2
|
|
%% and erl_compiler_opts_set/0 in needed_files
|
|
application:load(compiler),
|
|
_ = code:ensure_loaded(compile),
|
|
_ = code:ensure_loaded(Compiler),
|
|
ok.
|
|
|
|
run(G, CompilerMod, AppInfo, Contexts) ->
|
|
Name = rebar_app_info:name(AppInfo),
|
|
#{src_dirs := SrcDirs,
|
|
src_ext := SrcExt,
|
|
out_mappings := Mappings} = maps:get(Name, Contexts),
|
|
|
|
BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
|
|
|
|
BaseOpts = rebar_app_info:opts(AppInfo),
|
|
FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
|
|
|
|
{{FirstFiles, FirstFileOpts},
|
|
{RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),
|
|
|
|
Tracked =
|
|
compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod)
|
|
++ case RestFiles of
|
|
{Sequential, Parallel} -> % parallelizable form
|
|
compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod) ++
|
|
lists:append(
|
|
compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod)
|
|
);
|
|
_ when is_list(RestFiles) -> % traditional sequential build
|
|
compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod)
|
|
end,
|
|
store_artifacts(G, Tracked).
|
|
|
|
compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
|
|
[];
|
|
compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
|
|
case erlang:function_exported(CompilerMod, compile_and_track, 4) of
|
|
false ->
|
|
do_compile(CompilerMod, Source, Outs, Config, Opts),
|
|
compile_each(Rest, Opts, Config, Outs, CompilerMod);
|
|
true ->
|
|
do_compile_and_track(CompilerMod, Source, Outs, Config, Opts)
|
|
++ compile_each(Rest, Opts, Config, Outs, CompilerMod)
|
|
end.
|
|
|
|
do_compile(CompilerMod, Source, Outs, Config, Opts) ->
|
|
case CompilerMod:compile(Source, Outs, Config, Opts) of
|
|
ok ->
|
|
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
|
|
{ok, Warnings} ->
|
|
report(Warnings),
|
|
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
|
|
skipped ->
|
|
?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
|
|
Error ->
|
|
NewSource = format_error_source(Source, Config),
|
|
?ERROR("Compiling ~ts failed", [NewSource]),
|
|
maybe_report(Error),
|
|
?DEBUG("Compilation failed: ~p", [Error]),
|
|
?ABORT
|
|
end.
|
|
|
|
do_compile_and_track(CompilerMod, Source, Outs, Config, Opts) ->
|
|
case CompilerMod:compile_and_track(Source, Outs, Config, Opts) of
|
|
{ok, Tracked} ->
|
|
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
Tracked;
|
|
{ok, Tracked, Warnings} ->
|
|
report(Warnings),
|
|
?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
Tracked;
|
|
skipped ->
|
|
?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
[];
|
|
Error ->
|
|
NewSource = format_error_source(Source, Config),
|
|
?ERROR("Compiling ~ts failed", [NewSource]),
|
|
maybe_report(Error),
|
|
?DEBUG("Compilation failed: ~p", [Error]),
|
|
?ABORT
|
|
end.
|
|
|
|
store_artifacts(_G, []) ->
|
|
ok;
|
|
store_artifacts(G, [{Source, Target, Meta}|Rest]) ->
|
|
%% Assume the source exists since it was tracked to be compiled
|
|
rebar_compiler_dag:store_artifact(G, Source, Target, Meta),
|
|
store_artifacts(G, Rest).
|
|
|
|
compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) ->
|
|
Tracking = erlang:function_exported(CompilerMod, compile_and_track, 4),
|
|
rebar_parallel:queue(
|
|
Targets,
|
|
fun compile_worker/2, [Opts, BaseOpts, Mappings, CompilerMod],
|
|
fun compile_handler/2, [BaseOpts, Tracking]
|
|
).
|
|
|
|
compile_worker(Source, [Opts, Config, Outs, CompilerMod]) ->
|
|
Result = case erlang:function_exported(CompilerMod, compile_and_track, 4) of
|
|
false ->
|
|
CompilerMod:compile(Source, Outs, Config, Opts);
|
|
true ->
|
|
CompilerMod:compile_and_track(Source, Outs, Config, Opts)
|
|
end,
|
|
%% Bundle the source to allow proper reporting in the handler:
|
|
{Result, Source}.
|
|
|
|
compile_handler({ok, Source}, _Args) ->
|
|
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
ok;
|
|
compile_handler({{ok, Tracked}, Source}, [_, Tracking]) when Tracking ->
|
|
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
{ok, Tracked};
|
|
compile_handler({{ok, Warnings}, Source}, _Args) ->
|
|
report(Warnings),
|
|
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
ok;
|
|
compile_handler({{ok, Tracked, Warnings}, Source}, [_, Tracking]) when Tracking ->
|
|
report(Warnings),
|
|
?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
{ok, Tracked};
|
|
compile_handler({skipped, Source}, _Args) ->
|
|
?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), filename:basename(Source)]),
|
|
ok;
|
|
compile_handler({Error, Source}, [Config | _Rest]) ->
|
|
NewSource = format_error_source(Source, Config),
|
|
?ERROR("Compiling ~ts failed", [NewSource]),
|
|
maybe_report(Error),
|
|
?ABORT.
|
|
|
|
clean_(CompilerMod, AppInfo, _Label) ->
|
|
#{src_dirs := SrcDirs,
|
|
src_ext := SrcExt} = CompilerMod:context(AppInfo),
|
|
BaseDir = rebar_app_info:dir(AppInfo),
|
|
Opts = rebar_app_info:opts(AppInfo),
|
|
|
|
FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts),
|
|
CompilerMod:clean(FoundFiles, AppInfo),
|
|
ok.
|
|
|
|
annotate_extras(AppInfo) ->
|
|
AppOpts = rebar_app_info:opts(AppInfo),
|
|
ExtraDirs = rebar_dir:extra_src_dirs(AppOpts, []),
|
|
OldSrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]),
|
|
%% Re-annotate the directories with non-default options if it is the
|
|
%% case; otherwise, later down the line, the options get dropped with
|
|
%% profiles. All of this must be done with the rebar_dir functionality
|
|
%% which properly tracks and handles the various legacy formats for
|
|
%% recursion setting (erl_opts vs. dir options and profiles)
|
|
ExtraDirsOpts = [case rebar_dir:recursive(AppOpts, Dir) of
|
|
false -> {Dir, [{recursive, false}]};
|
|
true -> Dir
|
|
end || Dir <- ExtraDirs],
|
|
OldSrcDirsOpts = [case rebar_dir:recursive(AppOpts, Dir) of
|
|
false -> {Dir, [{recursive, false}]};
|
|
true -> Dir
|
|
end || Dir <- OldSrcDirs],
|
|
AppDir = rebar_app_info:dir(AppInfo),
|
|
lists:map(fun({DirOpt, Dir}) ->
|
|
EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
|
|
%% need a unique name to prevent lookup issues that clobber entries
|
|
AppName = unicode:characters_to_binary(
|
|
[rebar_app_info:name(AppInfo), "_", Dir]
|
|
),
|
|
AppInfo0 = rebar_app_info:name(AppInfo, AppName),
|
|
AppInfo1 = rebar_app_info:ebin_dir(AppInfo0, EbinDir),
|
|
AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [DirOpt]),
|
|
AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, OldSrcDirsOpts),
|
|
add_to_includes( % give access to .hrl in app's src/
|
|
AppInfo3,
|
|
[filename:join([AppDir, D]) || D <- OldSrcDirs]
|
|
)
|
|
end,
|
|
[T || T = {_DirOpt, ExtraDir} <- lists:zip(ExtraDirsOpts, ExtraDirs),
|
|
filelib:is_dir(filename:join(AppDir, ExtraDir))]
|
|
).
|
|
|
|
find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
|
|
SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$],
|
|
lists:flatmap(fun(SrcDir) ->
|
|
Recursive = rebar_dir:recursive(Opts, SrcDir),
|
|
rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive)
|
|
end, SrcDirs).
|
|
|
|
add_to_includes(AppInfo, Dirs) ->
|
|
Opts = rebar_app_info:opts(AppInfo),
|
|
List = rebar_opts:get(Opts, erl_opts, []),
|
|
NewErlOpts = [{i, Dir} || Dir <- Dirs] ++ List,
|
|
NewOpts = rebar_opts:set(Opts, erl_opts, NewErlOpts),
|
|
rebar_app_info:opts(AppInfo, NewOpts).
|
|
|
|
default_ctx() ->
|
|
#{dependencies_opts => []}.
|