#!/usr/bin/env escript %%! +A0 %% -*- coding: utf-8 -*- -mode(compile). -compile(nowarn_unused_function). -compile(nowarn_unused_vars). -export([main/1]). -define(SEP(I, C, L), <<(binary:copy(<<$%>>, I))/binary, (binary:copy(C, L))/binary>>). main(_) -> _ = io:setopts(user, [{encoding, unicode}]), check_ag_available(), ScriptsDir = filename:dirname(escript:script_name()), ok = file:set_cwd(filename:absname(ScriptsDir ++ "/..")), {Year, _, _} = erlang:date(), io:format("Edocify Kazoo...~n~n"), Run = [ %% regex for evil spec+specs {"ag -G '(erl|erl.src|hrl|hrl.src|escript)$' --nogroup '^\\-spec[^.]+\\.$(\\n+\\-spec[^.]+\\.$)+' core/ applications/" ,"separate evil sepc+specs" ,fun evil_specs/1 } ,{"find applications/ core/ -regextype egrep -regex '.*/src/.*.(erl|erl.src)$'" ,"bump/fix copyright" ,fun(R) -> bump_copyright(R, Year) end } %% regex to find contributors tag. ,{"ag -G '(erl|erl.src|hrl|hrl.src|escript)$' -l '%%+ *@?([Cc]ontributors|[Cc]ontributions)' core/ applications/" ,"rename contributors tag to author" ,fun edocify_headers/1 } %% regex to find `@public' tag. %% TODO: remove private tag from this regex in a distant future. ,{"ag -G '(erl|erl.src|hrl|hrl.src|escript)$' '%%* *@(public|private)' core/ applications/" ,"remove public/private tag" ,fun remove_public_tag/1 } %% regex for spec tag in comments: any comments which starts with `@spec' follow by anything (optional one time new line) %% until it ends (for single line @spec) any ending with `)' or `}' or any string at the end of the line (should be last regex otherwise %% multi line regex won't work). For multi line the first line should end with `|' followed by same regex until exhausted. ,{"ag '^%%+\\s*@spec((.*$\\n)?(.*\\)$|.*}$|.*\\|(\\n%%+(.*\\)$|.*}$|.*\\||[^@=-]+$))+)|.*$)' core/ applications/" ,"removing spec from comment" ,fun remove_comment_specs/1 } %% regex to find functions without comment block before them after a separator comment block %% to avoid EDoc to use the separator as the functions comment. %% Regex explanation: search for any line starts with at least two `%%' followed by any whitespace, followed by any new line until %% a `-spec' attribute or a function head is found. ,{"ag -G '(erl)$' '%%%*\\s*==+$(\\n+(^-spec+|[a-z]+))' applications/ core/" ,"add missing comments block after separator" ,fun missing_comment_blocks_after_sep/1 } %% regex for escaping codes in comment for `resource_exists' function crossbar modules. ,{"ag '%%%*\\s*Does the path point to a valid resource$(\\n%%*\\s*.*)*\\n%%%*\\s*@end' applications/" ,"escape code block in 'resource_exists' function crossbar modules" ,fun cb_resource_exists_comments/1 } %% regex for finding comment block with no @end ,{"ag '^%%*[ ]*@doc[^\\n]*$(\\n^(?!(%%* *@end|%%* ?--+$|%%* ?==+$))^%%[^\\n]*$)*(\\n%%* ?(--+|==+)$)' core/ applications/" ,"fix comment blocks with no end" ,fun comment_blocks_with_no_end/1 } %% regex for separator lines with length lower than 78 (for %%) or 77 (for %%%). ,{"ag -G '(applications|core)/.*/src/.*.(erl|erl.src|hrl|hrl.src)$' '^%% *-{50,77}$'" ,"increase separator line (starts with %%) length" ,fun(R) -> increase_sep_length(R, <<"-">>) end } ,{"ag -G '(applications|core)/.*/src/.*.(erl|erl.src|hrl|hrl.src)$' '^%%%+ *={50,76}$'" ,"increase separator line (starts with %%%) length" ,fun(R) -> increase_sep_length(R, <<"=">>) end } %% regex for finding first comment line after `@doc' ,{"ag '%%*\\s*@doc$(\\n%%*$)*\\n%%*\\s*[^@\\n]+$' core/ applications/" ,"move first comment line to the same line as doc tag" ,fun move_to_doc_line/1 } %% regex for empty comment line after @doc to avoid empty paragraph or dot in summary %% must be last thing to run ,{"ag -G '(erl|erl.src|hrl|hrl.src)$' '%%* *@doc *$(\\n%%* *$)+' core/ applications/" ,"remove empty comment line after doc tag" ,fun remove_doc_tag_empty_comment/1 } ], edocify(Run, 0). check_ag_available() -> case os:find_executable("ag") of false -> io:format("~nPlease install 'ag' (https://github.com/ggreer/the_silver_searcher):~n~n"), io:format(" apt-get install silversearcher-ag~n"), io:format(" brew install the_silver_searcher~n"), io:format(" yum install the_silver_searcher~n"), io:format(" pacman -S the_silver_searcher~n"), io:format("~n"), halt(1); _ -> ok end. run_ag(Cmd) -> try os:cmd(Cmd) catch _E:_T -> io:format("ag command failed: ~p:~p~n", [_E, _T]), halt(1) end. edocify([], 0) -> io:format("~nAlready EDocified! 🎉~n"); edocify([], Ret) -> io:format("~nWe had some EDocification! 🤔~n"), halt(Ret); edocify([{Cmd, Desc, Fun}|Rest], Ret) -> io:format("* running command: ~s~n", [Desc]), case check_result(list_to_binary(run_ag(Cmd))) of ok -> edocify(Rest, Ret); AgResult -> try Fun(AgResult) of ok -> io:format("\e[32;1mdone\e[0m~n"), edocify(Rest, 1); NewRet -> io:format("\e[32;1mdone\e[0m~n"), edocify(Rest, NewRet) catch _T:_R -> io:format("~nexception occurred while running command: ~p:~p~n", [_T, _R]), halt(1) end end. check_result(<<>>) -> io:format("\e[32;1mdone\e[0m~n"); check_result(<<"ERR:", _/binary>>=Error) -> io:format("ag command failed~n"), io:put_chars(Error), halt(1); check_result(Result) -> Result. %%------------------------------------------------------------------------------ %% @doc %% Moving grouped `spec' attributes to their own function header. %% Ag regex matches the lines starts with `-spec' to end of %% the it (by a single `.') including multi line spec. %% %% For multi line we accumulate it for that specific `spec' in a map %% and then first we read source file AST to find spec functions name %% and arity. Then we remove the evil specs from the file. %% %% Then we read AST again to find the function/arity position in the file %% then adding spec to line before appropriate function header. %% %% Ag sample output: %% ``` %% core/kazoo_apps/src/kapps_util.erl:313:-spec get_account_mods(kz_term:ne_binary()) -> kz_term:ne_binaries(). %% core/kazoo_apps/src/kapps_util.erl:314:-spec get_account_mods(kz_term:ne_binary(), kz_util:account_format()) -> kz_term:ne_binaries(). %% core/kazoo_apps/src/kapps_util.erl:608:-spec amqp_pool_request(kz_term:api_terms(), kz_amqp_worker:publish_fun(), kz_amqp_worker:validate_fun()) -> %% core/kazoo_apps/src/kapps_util.erl:609: kz_amqp_worker:request_return(). %% core/kazoo_apps/src/kapps_util.erl:610:-spec amqp_pool_request(kz_term:api_terms(), kz_amqp_worker:publish_fun(), kz_amqp_worker:validate_fun(), timeout()) -> %% core/kazoo_apps/src/kapps_util.erl:611: kz_amqp_worker:request_return(). %% ''' %% %% Expected outcome: %% move spec to appropriate function header. %% @end %%------------------------------------------------------------------------------ evil_specs(Result) -> FilesSpecs = map_specs_to_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], []), _ = lists:map(fun process_evil_specs/1, FilesSpecs), 'ok'. map_specs_to_file([], Acc) -> Acc; map_specs_to_file([H|T], SpecAcc) -> FirstLine = {File, _, _, true} = parse_ag_line(H), {NewTail, FileSpecs} = get_specs_for_file(File, T, [FirstLine]), FilePath = binary_to_list(File), {ok, Forms} = epp_dodger:quick_parse_file(FilePath, [{no_fail, true}]), SpecMaps = get_specs_info(File, Forms, FileSpecs, []), map_specs_to_file(NewTail, [{FilePath, SpecMaps}|SpecAcc]). %% Get all specs for this file. get_specs_for_file(_, [], Acc) -> {[], Acc}; get_specs_for_file(File, [H|T], Acc) -> case parse_ag_line(H) of {File, _, _, _}=Line -> get_specs_for_file(File, T, Acc ++ [Line]); {_, _, _, _} -> {[H|T], Acc} end. %% Find spec function name/arity from AST. get_specs_info(_, _, [], Acc) -> Acc; get_specs_info(File, Forms, [{_, Pos, Line, true}|T], Acc) -> %% {attribute, line, {{fun_name, arity}, func_type_ast}} {_, _, _, {{FunName, Arity}, _}} = lists:keyfind(Pos, 2, Forms), {NewTail, SpecLines} = get_multi_line_spec(T, [Line]), SpecMap = #{spec => SpecLines ,spec_pos => Pos ,spec_length => length(SpecLines) ,fun_name => FunName ,arity => Arity ,file => File }, get_specs_info(File, Forms, NewTail, Acc ++ [SpecMap]). %% Accumulate lines until next spec definition. get_multi_line_spec([{_, _, Line, false}|T], Acc) -> get_multi_line_spec(T, Acc ++ [Line]); get_multi_line_spec(Lines, Acc) -> {Lines, Acc}. %% Removing evil specs first then get functions new position from AST %% and add specs to position right before their function's position. process_evil_specs({File, SpecMaps}) -> io:format("processing ~s~n", [File]), remove_evil_specs(File, SpecMaps), {ok, Forms} = epp_dodger:quick_parse_file(File, [{no_fail, true}]), Lines = read_lines(File, false), save_lines(File, move_file_specs(Lines, 0, functions_new_position(Forms, SpecMaps, []))). %% First read files and create tuple List of {LineNumber, String} %% Then loop over SpecMaps and get the position of all lines that the spec %% is occupied. remove_evil_specs(File, SpecMaps) -> Lines = read_lines(File, true), Positions = lists:foldl(fun fetch_specs_pos/2, [], SpecMaps), save_lines(File, [L || {LN, L} <- Lines, not lists:member(LN, Positions)]). %% Extract specs positions (if it's multi line spec, add number of all lines it occupied). fetch_specs_pos(#{spec_pos := Pos, spec_length := Length}, Acc) -> Acc ++ [Pos + I - 1 || I <- lists:seq(1, Length)]. %% Find out the new position of the functions by looking fun/arity in AST. functions_new_position(_, [], Acc) -> lists:sort(fun sort_by_fun_position/2, Acc); functions_new_position(Forms, [#{fun_name := FunName, arity := Arity}=H|T], Acc) -> Functions = lists:filter(fun(Form) -> fun_arity_filter(Form, FunName, Arity) end, Forms), functions_new_position(Forms, T, Acc ++ is_test_or_regular_function(Functions, H, [])). fun_arity_filter({function, _Line, FName, Arity, _Clauses}, FName, Arity) -> true; fun_arity_filter(_, _, _) -> false. %% When there are two implementations for function, one normal and one guarded by `?TEST', %% save position of both functions to move spec to both positions. is_test_or_regular_function([], #{fun_name := FunName, arity := Arity, file := File}, []) -> io:format("~nCan't find function '~s/~b' in file '~s'~nIf this is header file remove the specs line from it and write specs in module itself." ,[FunName, Arity, File] ), halt(1); is_test_or_regular_function([], _, Acc) -> Acc; is_test_or_regular_function([{_, FunLine, _, _, _}|T], Map, Acc) -> is_test_or_regular_function(T, Map, [Map#{fun_pos => FunLine}|Acc]). %% This important since we are adding function starting from top line to last line. sort_by_fun_position(#{fun_pos := Pos1}, #{fun_pos := Pos2}) -> Pos1 < Pos2. %% Split file lines list at the position %% of the function + number of lines that we have added to file so far, %% then subtract it by one to get the function line in the right hand side list. %% For adding spec, first add an empty line before spec if it's necessary and adds everything together. %% At the end sum the number of added lines so far with current spec number of the line and possible empty line. move_file_specs(Lines, _, []) -> Lines; move_file_specs(Lines, LinesAdded, [#{fun_pos := Pos, spec := Spec, spec_length := Length}|T]) -> {Left, [Fun|Right]} = lists:split(Pos + LinesAdded - 1, Lines), EmptyLine = maybe_add_empty_line_before_spec(lists:last(Left)), move_file_specs(Left ++ EmptyLine ++ Spec ++ [Fun] ++ Right ,LinesAdded + Length + length(EmptyLine) ,T ). %%------------------------------------------------------------------------------ %% @doc Bump copyright year. Also add module header if it is missing (obviously only %% for files returned by `ag' not all other files). %% %% Ag sample output: %% ``` %% core/kazoo_apps/src/kapps_util.erl %% core/kazoo_voicemail/src/kvm_message.erl %% ''' %% %% Expected outcome: %% * Bump the year to current year %% * Have formal format of copyright line %% * Add blank module header if it is missing %% @end %%------------------------------------------------------------------------------ -define(DONT_BUMP, [<<"core/kazoo_stdlib/src/kz_mochinum.erl">> ,<<"core/gcm/src/gcm.erl">> ,<<"core/gcm/src/gcm_api.erl">> ,<<"core/gcm/src/gcm_sup.erl">> ,<<"core/gcm/src/gcm_app.erl">> ,<<"core/amqp_cron/src/amqp_cron_sup.erl">> ,<<"core/amqp_cron/src/amqp_cron_app.erl">> ,<<"core/amqp_cron/src/amqp_cron_task.erl">> ,<<"core/amqp_cron/src/amqp_cron.erl">> ,<<"core/kazoo_ast/src/kz_edoc_layout.erl">> ]). bump_copyright(Result, Y) -> Year = integer_to_binary(Y), Files = [F || F <- binary:split(Result, <<"\n">>, [global]), F =/= <<>>, not lists:member(F, ?DONT_BUMP) ], Bumped = [bump_copyright_file(F, Year) || F <- Files], case [OkBump || OkBump <- Bumped, OkBump =:= ok] of [] -> 0; _ -> 1 end. bump_copyright_file(File, Year) -> Lines = read_lines(File, false), {Module, Header, OtherLines} = get_module_header_comments(Lines, [], []), case re:run(iolist_to_binary(Header), "2600[Hh]z", [global]) of {match, _} -> bump_copyright_file(File, Module, Header, OtherLines, Year); _ -> case re:run(iolist_to_binary(Header), "@copyright", [global]) of {match, _} -> ignore; _ -> bump_copyright_file(File, Module, Header, OtherLines, Year) end end. bump_copyright_file(File, Module, Header, OtherLines, Year) -> MaybeBumped = bump_copyright(Module, Header, [], [], Year), case is_bumped(Header, MaybeBumped, Module) of true -> ignore; false -> io:format("processing ~s~n", [File]), save_lines(File, MaybeBumped ++ OtherLines) end. is_bumped(OldHeader, NewHeader, Module) -> OldHeader ++ Module =:= [strip_comment(H) || H <- NewHeader]. bump_copyright(Module, [], Copyright, [], Year) -> bump_copyright(Module, [], Copyright, [<<"%%% @doc">>], Year); bump_copyright(Module, [], [], Header, Year) -> bump_copyright(Module, [], generate_copyright_line(Year, <<>>), Header, Year); bump_copyright(Module, [], Copyright, Header, _) -> [?SEP(3, <<$->>, 77)] ++ Copyright ++ Header ++ [<<"%%% @end">>, ?SEP(3, <<$->>, 77)] ++ Module; bump_copyright(Module, [<<"@copyright", Rest/binary>>|T], _, Header, Year) -> bump_copyright(Module, T, do_bump_copyright(Rest, Year), Header, Year); bump_copyright(Module, [H|T], Copyright, Header, Year) -> Striped = strip_right_spaces(strip_left_spaces(H)), case Striped =/= <<"@end">> andalso is_separator_chars(Striped, [<<$=>>, <<$->>]) of false -> %% removing end tag to add it later bump_copyright(Module, T, Copyright, Header, Year); {true, _} -> %% removing separator to replace later bump_copyright(Module, T, Copyright, Header, Year); {false, _} when H =:= <<>> -> %% to avoid add whitespace if it is an empty comment line. bump_copyright(Module, T, Copyright, Header ++ [<<"%%%">>], Year); {false, _} -> bump_copyright(Module, T, Copyright, Header ++ [<<"%%% ", H/binary>>], Year) end. do_bump_copyright(C, Year) -> Nums = re:replace(C, "([^0-9\\-]*|2600)", <<>>, [global, {return, binary}]), case lists:usort([B || B <- binary:split(Nums, <<"-">>, [global]), B =/= <<>>]) of [Year] -> generate_copyright_line(Year, <<>>); [<<"20", _:2/binary>> = Y] -> generate_copyright_line(Y, Year); [<<"20", _:2/binary>> = Y, Year] -> generate_copyright_line(Y, Year); [<<"20", _:2/binary>> = Y, _] -> generate_copyright_line(Y, Year); _ -> generate_copyright_line(Year, <<>>) end. generate_copyright_line(StartY, EndY) -> [<<"%%% @copyright (C) ", StartY/binary, "-", EndY/binary, ", 2600Hz">>]. %%------------------------------------------------------------------------------ %% @doc %% Edocify Header by rename @contributors to @author. %% Ag will return a list of files and then we open each file and fix their %% header comment. %% * `-module' line should be immediately after header comment %% %% Ag sample output: %% ``` %% core/kazoo_apps/src/kapps_util.erl %% core/kazoo_voicemail/src/kvm_message.erl %% ''' %% %% Expected outcome: %% * create an `@author' tag for all names after `@contributors' %% or `@contributors'. (if any name at all) %% * all header comment lines (all comment lines before `-module' line or %% non comment line (non empty)) will start with `%%%' %% * header separator character will be `=' %% * will add `@end' tag if it's not there %% @end %%------------------------------------------------------------------------------ edocify_headers(Result) -> Files = [F || F <- binary:split(Result, <<"\n">>, [global]), F =/= <<>>], _ = [edocify_header(F) || F <- Files], 'ok'. edocify_header(File) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, false), {Module, Header, OtherLines} = get_module_header_comments(Lines, [], []), save_lines(File, edocify_header(Module, Header, []) ++ OtherLines). edocify_header(Module, [], Header) -> [?SEP(3, <<$->>, 77)] ++ Header ++ [<<"%%% @end">>, ?SEP(3, <<$->>, 77)] ++ Module; edocify_header(Module, [<<"@contributors", _/binary>>|T], Header) -> Authors = [<<"%%% @author ", Author/binary>> || A <- T, Author <- [strip_right_spaces(strip_left_spaces(A))], Author =/= <<>>, <<"@end">> =/= Author, not is_separator_char(Author, <<$->>), not is_separator_char(Author, <<$=>>) ], edocify_header(Module, [], Header ++ [<<"%%%">>] ++ Authors); edocify_header(Module, [<<"Contributors", Rest/binary>>|T], Header) -> %% it's capital `C' edocify_header(Module, [<<"@contributors", " ", Rest/binary>>|T], Header); edocify_header(Module, [<<"@contributions", Rest/binary>>|T], Header) -> %% mind you it is `contributions' not `contributors' edocify_header(Module, [<<"@contributors", " ", Rest/binary>>|T], Header); edocify_header(Module, [<<"@Contributions", Rest/binary>>|T], Header) -> %% mind you it is `Contributions' not `Contributors' edocify_header(Module, [<<"@contributors", " ", Rest/binary>>|T], Header); edocify_header(Module, [H|T], Header) -> Striped = strip_right_spaces(strip_left_spaces(H)), case Striped =/= <<"@end">> andalso is_separator_chars(Striped, [<<$=>>, <<$->>]) of false -> %% removing end tag to add it later edocify_header(Module, T, Header); {true, _} -> %% removing separator to replace later edocify_header(Module, T, Header); {false, _} when H =:= <<>> -> edocify_header(Module, T, Header ++ [<<"%%%">>]); {false, _} -> edocify_header(Module, T, Header ++ [<<"%%% ", H/binary>>]) end. get_module_header_comments([], Module, Header) -> {Module, Header}; get_module_header_comments([<<"%", _/binary>>=H | Lines], Module, Header) -> get_module_header_comments(Lines, Module, Header ++ [strip_right_spaces(strip_comment(H))]); get_module_header_comments([<<"-module", _/binary>>=Mod | Lines], _, Header) -> get_module_header_comments(Lines, [Mod], Header); get_module_header_comments([<<>>, <<"-module", _/binary>>=Mod| Lines], _, Header) -> get_module_header_comments(Lines, [Mod], Header); get_module_header_comments(Lines, Module, Header) -> {Module, Header, Lines}. %%------------------------------------------------------------------------------ %% @doc %% Removing `@public' from comments. %% Ad regex will match line with `@public'. %% So we can simply get file and positions for each file and remove the lines. %% %% Ag sample output: %% ``` %% applications/skel/src/skel_listener.erl:111:%% @public %% ''' %% %% Expected outcome: %% * all found lines should be removed. %% @end %%------------------------------------------------------------------------------ remove_public_tag(Result) -> CommentSpecs = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), _ = maps:map(fun do_remove_public_tag/2, CommentSpecs), 'ok'. do_remove_public_tag(File, Positions) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, [L || {LN, L} <- Lines, not lists:member(LN, Positions)]). %%------------------------------------------------------------------------------ %% @doc %% Removing @spec from comments. %% Ad regex will match line starts with `@spec' and the line it ends. So %% we can simply get file and positions for each file and remove the lines. %% %% Ag sample output: %% ``` %% applications/skel/src/skel_listener.erl:111:%% @spec handle_info(Info, State) -> {noreply, State} | %% applications/skel/src/skel_listener.erl:112:%% {noreply, State, Timeout} | %% applications/skel/src/skel_listener.erl:113:%% {stop, Reason, State} %% applications/skel/src/skel_listener.erl:122:%% @spec handle_event(JObj, State) -> {reply, Options} %% applications/skel/src/skel_listener.erl:136:%% @spec terminate(Reason, State) -> void() %% ''' %% %% Expected outcome: %% * all found lines should be removed. %% @end %%------------------------------------------------------------------------------ remove_comment_specs(Result) -> CommentSpecs = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), _ = maps:map(fun do_remove_comment_specs/2, CommentSpecs), 'ok'. do_remove_comment_specs(File, Positions) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, [L || {LN, L} <- Lines, not lists:member(LN, Positions)]). %%------------------------------------------------------------------------------ %% @doc %% Missing comment block after separator. %% Ad regex will match lines which starts with comment separator and ends with %% the line which has a `-spec' attribute or function head. We then go to those %% positions and format those lines. %% %% Ag sample output: %% ``` %% core/kazoo_media/src/kz_media_file_cache.erl:243:%%%============================================================================= %% core/kazoo_media/src/kz_media_file_cache.erl:244:-spec start_timer() -> reference(). %% core/kazoo_media/src/kz_media_map.erl:311:%%%============================================================================= %% core/kazoo_media/src/kz_media_map.erl:312: %% core/kazoo_media/src/kz_media_map.erl:313:-spec init_map() -> 'ok'. %% ''' %% %% Expected outcome: %% * an empty comment block before the spec line or function header (func without spec line). %% * maybe an empty comment line after the separator line if it's not exists. %% * any extra line between separator and `spec' line will be removed. %% @end %%------------------------------------------------------------------------------ missing_comment_blocks_after_sep(Result) -> Positions = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), _ = maps:map(fun add_missing_comment_blocks/2, Positions), 'ok'. add_missing_comment_blocks(File, Positions) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, do_add_missing_comment_blocks(Lines, Positions, [])). do_add_missing_comment_blocks([], _, Formatted) -> Formatted; do_add_missing_comment_blocks([{LN, Line}|Lines], Positions, Formatted) -> case lists:member(LN, Positions) andalso Line of false -> do_add_missing_comment_blocks(Lines, Positions, Formatted ++ [Line]); <<>> -> %% remove extra new lines do_add_missing_comment_blocks(Lines, Positions, Formatted); <<"-spec", _/binary>> -> %% add empty comment block do_add_missing_comment_blocks(Lines, Positions, Formatted ++ maybe_add_empty_line(lists:last(Formatted)) ++ empty_block() ++ [Line]); <<"%", _/binary>> -> %% maybe add empty line after separator case Lines of [] -> do_add_missing_comment_blocks(Lines, Positions, Formatted ++ [Line]); [<<>>|_] -> do_add_missing_comment_blocks(Lines, Positions, Formatted ++ [Line]); _ -> do_add_missing_comment_blocks(Lines, Positions, Formatted ++ [Line, <<>>]) end; <<"-", _/binary>> -> %% the idea was the regex should match any attribute line before `spec' but it change to match only spec %% so this is just left over from previous version and should not be run anyway. do_add_missing_comment_blocks(Lines, Positions, Formatted ++ [Line]); _ -> %% add empty comment block do_add_missing_comment_blocks(Lines, Positions, Formatted ++ maybe_add_empty_line(lists:last(Formatted)) ++ empty_block() ++ [Line]) end. %%------------------------------------------------------------------------------ %% @doc %% Escape codes in comment block for `resource_exists' function crossbar modules. %% Ag regex will match the comments before resource_exists function and last line %% is `%% @end'. We then formats those lines accordingly. %% %% Ag sample output: %% ``` %% applications/acdc/src/cb_queues.erl:143:%% Does the path point to a valid resource %% applications/acdc/src/cb_queues.erl:144:%% So /queues => [] %% applications/acdc/src/cb_queues.erl:145:%% /queues/foo => [<<"foo">>] %% applications/acdc/src/cb_queues.erl:146:%% /queues/foo/bar => [<<"foo">>, <<"bar">>] %% applications/acdc/src/cb_queues.erl:147:%% @end %% ''' %% %% Expected outcome: %% %% Does the path point to a valid resource. %% %% %% %% For example: %% %% %% %% ``` %% %% /queues => [] %% %% /queues/foo => [<<"foo">>] %% %% /queues/foo/bar => [<<"foo">>, <<"bar">>] %% %% ''' %% %% @end %% @end %%------------------------------------------------------------------------------ cb_resource_exists_comments(Result) -> Positions = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), _ = maps:map(fun fix_cb_resource_exists_comment/2, Positions), 'ok'. fix_cb_resource_exists_comment(File, Positions) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, do_cb_resource_exists_comment(Lines, Positions, [])). do_cb_resource_exists_comment([], _, Formatted) -> Formatted; do_cb_resource_exists_comment([{LN, Line}|Lines], Positions, Formatted) -> case lists:member(LN, Positions) andalso Line of false -> do_cb_resource_exists_comment(Lines, Positions, Formatted ++ [Line]); <<"%%">> -> %% remove empty comment line do_cb_resource_exists_comment(Lines, Positions, Formatted); <<"%% @end">> -> Formatted ++ [Line] ++ [L || {_, L} <- Lines]; <<"%% So ", Rest/binary>> -> do_cb_resource_exists_comment(Lines ,Positions ,Formatted ++ [<<"%%">>, <<"%% For example:">>, <<"%%">>, <<"%% ```">>, <<"%% ", Rest/binary, ".">>] ); _ -> case Lines of [{_, <<"%% @end">>}|_] -> do_cb_resource_exists_comment(Lines, Positions, Formatted ++ [Line, <<"%% '''">>]); _ -> do_cb_resource_exists_comment(Lines, Positions, Formatted ++ [Line]) end end. %%------------------------------------------------------------------------------ %% @doc %% Fix comment block which doesn't end properly with `@end' tag. %% %% Ag sample output: %% ``` %% applications/ecallmgr/src/ecallmgr_originate.erl:67:%% @doc Starts the server %% applications/ecallmgr/src/ecallmgr_originate.erl:68:%%-------------------------------------------------------------------- %% applications/ecallmgr/src/ecallmgr_originate.erl:128:%% @doc %% applications/ecallmgr/src/ecallmgr_originate.erl:129:%% Initializes the server %% applications/ecallmgr/src/ecallmgr_originate.erl:130:%%-------------------------------------------------------------------- %% ''' %% %% Expected outcome: %% * an `@end' tag should be added before the separator line. %% @end %%------------------------------------------------------------------------------ comment_blocks_with_no_end(Result) -> Positions = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), _ = maps:map(fun comment_blocks_with_no_end/2, Positions), 'ok'. comment_blocks_with_no_end(File, Positions) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, do_comment_blocks_with_no_end(Lines, Positions, [])). do_comment_blocks_with_no_end([], _, Formatted) -> Formatted; do_comment_blocks_with_no_end([{LN, Line}|Lines], Positions, Formatted) -> {PerCount, {IsSeprator, _}} = analyze_separator(Line, [<<$=>>, <<$->>]), case lists:member(LN, Positions) andalso IsSeprator of false -> do_comment_blocks_with_no_end(Lines, Positions, Formatted ++ [Line]); true -> do_comment_blocks_with_no_end(Lines, Positions, Formatted ++ [<<(binary:copy(<<$%>>, PerCount))/binary, " @end">>, Line]) end. %%------------------------------------------------------------------------------ %% @doc Increase separator line length. %% %% Ag sample output: %% `` %% applications/konami/src/konami_listener.erl:3:%%-------_more_--- %% applications/konami/src/konami_listener.erl:5:%%-------_more_--- %% ''' %% %% Expected outcome: %% * For `%%' separator, there should be 78 `-' %% * For `%%%' separator, there should be 77 `=' %% * for both cases any spaces between comment character and separator should be remove. %% @end %%------------------------------------------------------------------------------ increase_sep_length(Result, SepChar) -> Positions = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), Separator = make_me_sep(SepChar), _ = maps:map(fun(F, P) -> increase_sep_length(F, P, Separator) end, Positions), 'ok'. make_me_sep(S = <<"-">>) -> ?SEP(2, S, 78); make_me_sep(S = <<"=">>) -> ?SEP(3, S, 77). increase_sep_length(File, Positions, Separator) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, do_increase_sep_length(Lines, Positions, Separator, [])). do_increase_sep_length([], _, _, Formatted) -> Formatted; do_increase_sep_length([{LN, Line}|Lines], Positions, Separator, Formatted) -> case lists:member(LN, Positions) of true -> do_increase_sep_length(Lines, Positions, Separator, Formatted ++ [Separator]); false -> do_increase_sep_length(Lines, Positions, Separator, Formatted ++ [Line]) end. %%------------------------------------------------------------------------------ %% @doc %% Move first comment line to the same line as `@doc'. So EDoc is not %% adding extra new line character and spaces to beginning of the %% paragraph. (Not particularly necessary but it makes better %% looking HTML code at the end). %% %% CAUTION: This code also makes sure there is no empty comment line %% between `@doc' line and separator to avoid inclusion of separator line %% in the documentation. %% %% Ag sample output: %% `` %% applications/konami/src/konami_listener.erl:3:%%% @doc %% applications/konami/src/konami_listener.erl:4:%%% %% applications/konami/src/konami_listener.erl:149:%% @doc %% applications/konami/src/konami_listener.erl:150:%% Initializes the server %% applications/konami/src/konami_listener.erl:164:%% @doc %% applications/konami/src/konami_listener.erl:165:%% Handling call messages %% ''' %% %% Expected outcome: %% * for example, %% %% @doc Initializes the server. %% * also removes empty comment lines after `@doc' and before first non empty comment line %% @end %%------------------------------------------------------------------------------ move_to_doc_line(Result) -> Positions = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), _ = maps:map(fun move_to_doc_line/2, Positions), 'ok'. move_to_doc_line(File, Positions) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, do_move_to_doc_line(Lines, Positions, [])). do_move_to_doc_line([], _, Formatted) -> Formatted; do_move_to_doc_line([{LN, Line}|Lines], Positions, Formatted) -> case lists:member(LN, Positions) andalso strip_right_spaces(strip_left_spaces(strip_comment(Line))) of false -> do_move_to_doc_line(Lines, Positions, Formatted ++ [Line]); <<>> -> %% remove empty comment line do_move_to_doc_line(Lines, Positions, Formatted); <<"@doc">> -> PerCount = count_percent(Line), %% add doc tag in case there is no non-empty comment lines so we don't loose the doc tag do_move_to_doc_line(Lines, Positions, Formatted ++ [<<(binary:copy(<<$%>>, PerCount))/binary, " @doc">>]); Rest -> %% this clause should only match once for the first non empty comment line PerCount = count_percent(Line), DocTagLine = <<(binary:copy(<<$%>>, PerCount))/binary, " @doc">>, NewForm = case lists:last(Formatted) of DocTagLine -> lists:droplast(Formatted); _ -> Formatted end, do_move_to_doc_line(Lines, Positions, NewForm ++ [<<(DocTagLine)/binary, " ", Rest/binary>>]) end. %%------------------------------------------------------------------------------ %% @doc %% Remove empty comment lines after `@doc' to avoid empty paragraph %% or dot in summary. Regex only returns the line with `@doc' and %% empty comment line. %% %% Ag sample output: %% ``` %% applications/stepswitch/src/stepswitch_util.erl:3:%%% @doc %% applications/stepswitch/src/stepswitch_util.erl:4:%%% %% applications/stepswitch/src/stepswitch_util.erl:25:%% @doc %% applications/stepswitch/src/stepswitch_util.erl:26:%% %% ''' %% %% Expected outcome: %% * removes all those empty comment line after `@doc'. %% @end %%------------------------------------------------------------------------------ remove_doc_tag_empty_comment(Result) -> Positions = collect_positions_per_file([Line || Line <- binary:split(Result, <<"\n">>, [global]), Line =/= <<>>], #{}), _ = maps:map(fun remove_doc_tag_empty_comment/2, Positions), 'ok'. remove_doc_tag_empty_comment(File, Positions) -> io:format("processing ~s~n", [File]), Lines = read_lines(File, true), save_lines(File, do_remove_doc_tag_empty_comment(Lines, Positions, [])). do_remove_doc_tag_empty_comment([], _, Formatted) -> Formatted; do_remove_doc_tag_empty_comment([{LN, Line}|Lines], Positions, Formatted) -> case lists:member(LN, Positions) andalso strip_right_spaces(strip_left_spaces(strip_comment(Line))) of false -> do_remove_doc_tag_empty_comment(Lines, Positions, Formatted ++ [Line]); <<>> -> %% remove empty comment line do_remove_doc_tag_empty_comment(Lines, Positions, Formatted); _ -> do_remove_doc_tag_empty_comment(Lines, Positions, Formatted ++ [Line]) end. %%%============================================================================= %%% Utilities %%%============================================================================= maybe_add_empty_line(<<>>) -> []; maybe_add_empty_line(_) -> [<<>>]. maybe_add_empty_line_before_spec(<<>>) -> []; %% don't add empty line if it's a comment block, maybe it has some corner case, %% e.g. the separator belongs to function comment block or not. But most of the %% time we're good to go this way. maybe_add_empty_line_before_spec(<<"%%--", _/binary>>) -> []; maybe_add_empty_line_before_spec(<<"%% --", _/binary>>) -> []; maybe_add_empty_line_before_spec(_) -> [<<>>]. analyze_separator(<<"%", _/binary>>=Line, SepChars) -> analyze_separator(Line, SepChars, 0); analyze_separator(_, _) -> {0, {false, []}}. analyze_separator(<<>>, _, _) -> {0, {false, []}}; analyze_separator(<<"%", Rest/binary>>, Chars, PerCount) -> analyze_separator(Rest, Chars, PerCount + 1); analyze_separator(Rest, Chars, PerCount) -> {PerCount, is_separator_chars(strip_right_spaces(strip_left_spaces(Rest)), Chars)}. is_separator_chars(_, []) -> {false, []}; is_separator_chars(Line, [H|T]) -> case is_separator_char(Line, H) of true -> {true, H}; false -> is_separator_chars(Line, T) end. is_separator_char(C, C) -> true; is_separator_char(<>, C) -> is_separator_char(B, C); is_separator_char(_, _) -> false. collect_positions_per_file([], Map) -> Map; collect_positions_per_file([Line | Lines], Map) -> {File, Pos, _, _} = parse_ag_line(Line), collect_positions_per_file(Lines, Map#{File => maps:get(File, Map, []) ++ [Pos]}). parse_ag_line(<<"ERR:", _/binary>>=Error) -> io:format("~nag command failed,~n"), io:put_chars(Error), halt(1); parse_ag_line(Bin) -> try explode_line(Bin) catch _:_ -> io:format("~nfailed to explode line in ag output, failed output line:~n~s~n", [Bin]), halt(1) end. explode_line(Bin) -> [File, PosRest] = binary:split(Bin, <<":">>), [Pos, Rest] = binary:split(PosRest, <<":">>), Line = iolist_to_binary(Rest), {File, binary_to_integer(Pos), Line, is_spec_line(Line)}. is_spec_line(<<"-spec", _/binary>>) -> true; is_spec_line(_) -> false. strip_comment(<<$%, B/binary>>) -> strip_comment(B); strip_comment(<<$\s, B/binary>>) -> B; strip_comment(A) -> A. count_percent(A) -> count_percent(A, 0). count_percent(<<$%, B/binary>>, Count) -> count_percent(B, Count + 1); count_percent(_, Count) -> Count. strip_left_spaces(<<$\s, B/binary>>) -> strip_left_spaces(B); strip_left_spaces(A) -> A. strip_right_spaces(<<$\s>>) -> <<>>; strip_right_spaces(<<$\s, B/binary>>) -> case strip_right_spaces(B) of <<>> -> <<>>; T -> <<$\s, T/binary>> end; strip_right_spaces(<>) -> <>; strip_right_spaces(<<>>) -> <<>>. empty_block() -> [<<"%%--------------------------------------------------------------------">> ,<<"%% @doc">> ,<<"%% @end">> ,<<"%%--------------------------------------------------------------------">> ]. read_lines(File, WithLineNumber) -> case file:read_file(File) of {ok, Bin} -> case WithLineNumber of true -> Fun = fun(L, {LN, Ls}) -> {LN+1, [{LN, L}|Ls]} end, Splits = binary:split(Bin, <<"\n">>, [global]), {_, Lines} = lists:foldl(Fun, {1, []}, Splits), lists:reverse(Lines); false -> binary:split(Bin, <<"\n">>, [global]) end; {error, Reason} -> io:format("failed to read lines from ~s: ~p~n", [File, Reason]), throw({error, Reason}) end. save_lines(File, Lines) -> Data = check_final_newline(lists:reverse([<> || L <- Lines])), case file:write_file(File, Data) of ok -> ok; {error, Reason} -> io:format("failed to write lines to ~s: ~p~n", [File, Reason]), throw({error, Reason}) end. check_final_newline([<<"\n">>|Tser]) -> check_final_newline(Tser); check_final_newline(Senil) -> lists:reverse(Senil).