From 3a3950a7d661ca048cbf78625685d49f54e9881e Mon Sep 17 00:00:00 2001 From: Kolan Sh Date: Mon, 9 Apr 2012 16:51:08 +0400 Subject: [PATCH] Scripts for Rhodecode and Doxygen --- chiliproject_doxygen.sh | 179 ++++++++++++++++++++++++++++++++++++++ rhodecode_chiliproject.sh | 119 +++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100755 chiliproject_doxygen.sh create mode 100755 rhodecode_chiliproject.sh diff --git a/chiliproject_doxygen.sh b/chiliproject_doxygen.sh new file mode 100755 index 0000000..0e8a8b3 --- /dev/null +++ b/chiliproject_doxygen.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +#VARS +DOC_PATH=/var/doc/redmine +TMP_PATH=/tmp/redmine-doxygen$RANDOM$RANDOM + +# table +project_id= +type= +root_url= +#id= # the same as project_id +identifier= + +# UMASK +umask 0002 + +# read redmine.repositories to table +MYSQL_RESULT=`mysql -h127.0.0.1 -u redmine -e "SELECT project_id, type, root_url FROM redmine.repositories" | grep -v tables_col|xargs|sed "s/ /\n/g"|tail -n+4` +let n=0 +for v in $MYSQL_RESULT; do + let idx=n/3 + case $((n%3)) in + 0) project_id[$idx]=$v;; + 1) type[$idx]=$v;; + 2) root_url[$idx]=$v;; + esac + let n++ +done; +let n/=3 + +# read redmine.projects to table +MYSQL_RESULT=`mysql -h127.0.0.1 -u redmine -e "SELECT id, identifier FROM redmine.projects" | grep -v tables_col|xargs|sed "s/ /\n/g"|tail -n+3` +last_idx=0 +let i=0 +for v in $MYSQL_RESULT; do + case $((i%2)) in + 0) last_idx=$v + ;; + 1) for j in `seq 0 $((n-1))`; do + if [ "$last_idx" == "${project_id[$j]}" ]; then + identifier[$j]=$v + break + fi + done + ;; + esac + let i++ +done + +# remove old documentation +cd $DOC_PATH +[ $? != 0 ] && echo "cd $DOC_PATH failed" && rm -rf $TMP_PATH && exit -1 +for d in *; do + let found=false + for i in `seq 0 $((n-1))`; do + if [[ "${identifier[$i]}" == "$d" ]]; then + found=true + break + fi + done + [ $found == false ] && rm -rf $d +done + +# generate documentation +mkdir $TMP_PATH && cd $TMP_PATH +[ $? != 0 ] && echo "mkdir $TMP_PATH && cd $TMP_PATH failed" && rm -rf $TMP_PATH && exit -1 +for i in `seq 0 $((n-1))`; do + cd $TMP_PATH + + # Checkout last tags from repos. If no tags exist then go to next cycle iteration. + LAST_TAG="" + case ${type[$i]} in + Mercurial|Repository::Mercurial) + LAST_TAG=`hg tags --color never --noninteractive --quiet -R ${root_url[i]} 2>/dev/null | head -n2 | grep -v '^tip$' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+'` + ;; + Git|Repository::Git) + cd ${root_url[$i]} + [ $? != 0 ] && echo "cd ${root_url[$i]} failed" && continue + LAST_TAG=`git tag | tail -n1 \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+'` + cd $TMP_PATH + ;; + esac + + # Continue if no tags found + [ "" == "$LAST_TAG" ] && echo "No tags found for project ${root_url[$i]}" && continue + + # If documentation exists for $LAST_TAG then continue + [ "`cat $DOC_PATH/${identifier[$i]}/tag 2>/dev/null`" == "$LAST_TAG" ] \ + && echo "Documentation alredy exists for ${identifier[$i]}" \ + && cd $TMP_PATH && continue + + + # Lock dir by creating tag file + mkdir -p $DOC_PATH/${identifier[$i]} 2>/dev/null + echo $LAST_TAG >$DOC_PATH/${identifier[$i]}/tag + [ $? != 0 ] && echo "echo $LAST_TAG >$DOC_PATH/${identifier[$i]}/tag failed" && rm -rf $TMP_PATH && rm -f $DOC_PATH/${identifier[$i]}/tag && exit -1 + + # GENERATING DOCUMENTATION + echo "Generating documentation for ${root_url[$i]}" + + # cp directory to $TMP_PATH + repo_dir_name=${root_url[i]%/} + repo_dir_name=${repo_dir_name##*/} + + # Clone and Checkout + case ${type[$i]} in + Mercurial|Repository::Mercurial) + hg clone ${root_url[i]} $TMP_PATH/$repo_dir_name && cd $TMP_PATH/$repo_dir_name + [ $? != 0 ] && echo "hg clone ${root_url[i]} $TMP_PATH/$repo_dir_name && cd $TMP_PATH/$repo_dir_name failed" && rm -rf $TMP_PATH && rm -f $DOC_PATH/${identifier[$i]}/tag && exit -1 + hg up -C $LAST_TAG + ;; + Git|Repository::Git) + git clone ${root_url[i]} $TMP_PATH/$repo_dir_name && cd $TMP_PATH/$repo_dir_name + [ $? != 0 ] && echo "git clone ${root_url[i]} $TMP_PATH/$repo_dir_name && cd $TMP_PATH/$repo_dir_name failed" && rm -rf $TMP_PATH && rm -f $DOC_PATH/${identifier[$i]}/tag && exit -1 + git checkout $LAST_TAG + ;; + esac + + # Converting Files to UTF-8 encoding + find $TMP_PATH/$repo_dir_name \( ! -regex '.*/\..*' \) -type f -exec detect_encoding_and_convert.sh utf-8 '{}' \; + + # Generate doxygen documentation + doxygen -g doxygen.conf + sed " + s~^PROJECT_NAME.*$~PROJECT_NAME = ~; + s~^OUTPUT_LANGUAGE.*$~OUTPUT_LANGUAGE = English~; + s~^BUILTIN_STL_SUPPORT.*$~BUILTIN_STL_SUPPORT = YES~; + s~^EXTRACT_ALL.*$~EXTRACT_ALL = YES~; + s~^EXTRACT_PRIVATE.*$~EXTRACT_PRIVATE = YES~; + s~^EXTRACT_STATIC.*$~EXTRACT_STATIC = YES~; + s~^EXTRACT_LOCAL_METHODS.*$~EXTRACT_LOCAL_METHODS = YES~; + s~^EXTRACT_ANON_NSPACES.*$~EXTRACT_ANON_NSPACES = YES~; + s~^FORCE_LOCAL_INCLUDES.*$~FORCE_LOCAL_INCLUDES = YES~; + s~^SHOW_DIRECTORIES.*$~SHOW_DIRECTORIES = YES~; + s~^RECURSIVE.*$~RECURSIVE = YES~; + s~^SOURCE_BROWSER.*$~SOURCE_BROWSER = YES~; + s~^VERBATIM_HEADERS.*$~VERBATIM_HEADERS = NO~; + s~^REFERENCED_BY_RELATION.*$~REFERENCED_BY_RELATION = YES~; + s~^REFERENCED_RELATION.*$~REFERENCED_RELATION = YES~; + s~^GENERATE_LATEX.*$~GENERATE_LATEX = NO~; + s~^HAVE_DOT.*$~HAVE_DOT = YES~; + s~^UML_LOOK.*$~UML_LOOK = YES~; + s~^TEMPLATE_RELATIONS.*$~TEMPLATE_RELATIONS = YES~; + s~^CALL_GRAPH.*$~CALL_GRAPH = YES~; + s~^CALLER_GRAPH.*$~CALLER_GRAPH = YES~; + s~^EXCLUDE_PATTERNS.*$~EXCLUDE_PATTERNS = .hg .git~; + s~^HTML_FOOTER.*$~HTML_FOOTER = footer.html~; + " -i doxygen.conf + + # project name and version in the footer + echo "
$repo_dir_name-$LAST_TAG" > footer.html + + # run doxygen generator + doxygen doxygen.conf + + # README in title page + README="`find -maxdepth 1 -type f -iname 'readme*' | head -n1`" + if [ -f "$README" ]; then + sed -i 's~http\(\|s\)\(://[^ \n\t]*\)~http\1\2~g' README + sed -i 's~$~
~' "$README" + sed -i "/
/r $README" html/index.html + fi + + # Copy html to $DOC_PATH + mkdir -p $DOC_PATH/${identifier[$i]} + [ $? != 0 ] && echo "mkdir -p $DOC_PATH/${identifier[$i]} failed" && rm -rf $TMP_PATH && rm -f $DOC_PATH/${identifier[$i]}/tag && exit -1 + rm -rf $DOC_PATH/${identifier[$i]}/html + cp -r html $DOC_PATH/${identifier[$i]} + + # remove temp dir + cd $TMP_PATH + rm -rf $TMP_PATH/* +done + +# remove $TMP_PATH +rm -rf $TMP_PATH + diff --git a/rhodecode_chiliproject.sh b/rhodecode_chiliproject.sh new file mode 100755 index 0000000..574daab --- /dev/null +++ b/rhodecode_chiliproject.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +# VARS +export LANG=ru_RU.UTF-8 +RHODECODE_SQLITE_PATH=/home/rhodecode/rhodecode/rhodecode.db +CHILI_MYSQL_USER=redmine +CHILI_MYSQL_DBNAME=redmine +CHILI_MYSQL_HOSTNAME=127.0.0.1 +CHILI_REQUIRED_ROLES="Ответственный Разработчик Менеджер Major Developer Manager" + +# === REMOVE ALL BROKEN REPOSITORY LINKS IN REDMINE MYSQL DATABASE === +ALL_MYSQL_REPOS=`mysql -h$CHILI_MYSQL_HOSTNAME -u $CHILI_MYSQL_USER -e "SELECT url,root_url,id FROM $CHILI_MYSQL_DBNAME.repositories WHERE type='Mercurial' OR type='Repository::Mercurial'" | grep -v tables_col|xargs|sed "s/ /\n/g"|tail -n+4` +repos_to_remove= +current_url= +current_root_url= +let n=0 +for v in $ALL_MYSQL_REPOS; do + let idx=n/3 + case $((n%3)) in + 0) current_url=$v;; + 1) current_root_url=$v;; + 2) [ ! -d $current_url -a ! -d $current_root_url ] && repos_to_remove="$repos_to_remove,$v";; + esac + let n++ +done; +repos_to_remove=${repos_to_remove#,} +[ "$repos_to_remove" != "" ] && mysql -h$CHILI_MYSQL_HOSTNAME -u $CHILI_MYSQL_USER -e "DELETE FROM $CHILI_MYSQL_DBNAME.repositories WHERE id IN ($repos_to_remove)" + +# === GET DATA FROM RHODECODE SQLITE BASE === +rh_repos_path=`sqlite3 $RHODECODE_SQLITE_PATH "select ui_value FROM rhodecode_ui where ui_section='paths'"` + +SQLITE_RESULTS=`sqlite3 $RHODECODE_SQLITE_PATH "SELECT repo_name,repo_type,users.username,users_groups.users_group_name + FROM repositories,users,users_groups,users_groups_members + WHERE repositories.user_id=users.user_id + AND users.user_id=users_groups_members.user_id + AND users_groups.users_group_id=users_groups_members.users_group_id;"` + +# initializing repos arrays and count them +repos_names= +repos_paths= +repos_types= +repos_users= +repos_groups= + +let nrepos=0 +for r in $SQLITE_RESULTS; do + repos_paths[$nrepos]=$rh_repos_path/${r%|*|*|*} + tmp=${repos_paths[$nrepos]%/}; repos_names[$nrepos]=${tmp##*/} + tmp=${r%|*|*}; repos_types[$nrepos]=${tmp#*|} + case ${repos_types[$nrepos]} in + hg) repos_types[$nrepos]='Mercurial';; + git) repos_types[$nrepos]='Git';; + esac + tmp=${r%|*}; repos_users[$nrepos]=${tmp#*|*|} + repos_groups[$nrepos]=${r#*|*|*|} + let nrepos++ +done + +# === FOR ALL REPOS FROM RHODECODE DATABASE=== +for i in `seq 0 $((nrepos-1))`; do + # === GET DATA FROM REDMINE MYSQL BASE === + ALREADY_EXIST=`mysql -h$CHILI_MYSQL_HOSTNAME -u $CHILI_MYSQL_USER -e "SELECT id + FROM $CHILI_MYSQL_DBNAME.repositories + WHERE url='${repos_paths[$i]}' + OR root_url='${repos_paths[$i]}'" \ + | grep -v tables_col|xargs|sed "s/ /\n/g"|tail -n+2` + [ "$ALREADY_EXIST" != "" ] && continue + + USERID=`mysql -h$CHILI_MYSQL_HOSTNAME -u $CHILI_MYSQL_USER -e "SELECT id + FROM $CHILI_MYSQL_DBNAME.users,$CHILI_MYSQL_DBNAME.groups_users + WHERE users.id=groups_users.user_id + AND users.status='1' + AND users.login='${repos_users[$i]}' + AND users.type='User' + AND groups_users.group_id=(SELECT id + FROM $CHILI_MYSQL_DBNAME.users + WHERE users.type='Group' + AND users.lastname='${repos_groups[$i]}' + AND users.status='1')" \ + | grep -v tables_col|xargs|sed "s/ /\n/g"|tail -n+2` + [ "$USERID" == "" ] && continue + + PROJECTID=`mysql -h$CHILI_MYSQL_HOSTNAME -u $CHILI_MYSQL_USER -e "SELECT id FROM $CHILI_MYSQL_DBNAME.projects + WHERE (name='${repos_names[$i]}' + OR identifier='${repos_names[$i]}') + AND status='1'" \ + | grep -v tables_col|xargs|sed "s/ /\n/g"|tail -n+2` + [ "$PROJECTID" == "" ] && continue + + roles_mysql_string=`echo $CHILI_REQUIRED_ROLES | sed "s~\>~'~g ; s~\<~OR roles.name='~g ; s~^OR ~~"` + ROLES=`mysql -h$CHILI_MYSQL_HOSTNAME -u $CHILI_MYSQL_USER -e "SELECT roles.name + FROM $CHILI_MYSQL_DBNAME.roles,$CHILI_MYSQL_DBNAME.member_roles,$CHILI_MYSQL_DBNAME.members + WHERE roles.id=member_roles.role_id + AND member_roles.member_id=members.id + AND members.user_id='$USERID' + AND members.project_id='$PROJECTID' + AND ($roles_mysql_string)" \ + | grep -v tables_col|xargs|sed "s/ /\n/g"|tail -n+2` + + [ "$ROLES" == "" ] && continue + + # === ATTACH RHODECODE REPOSITORY TO REDMINE === + # DEBUG + echo "insert $PROJECTID,${repos_paths[$i]},${repos_types[$i]}" + + mysql -h$CHILI_MYSQL_HOSTNAME -u $CHILI_MYSQL_USER -e "INSERT INTO $CHILI_MYSQL_DBNAME.repositories(project_id, + url, + root_url, + type, + path_encoding, + extra_info) + VALUES('$PROJECTID', + '${repos_paths[$i]}', + '${repos_paths[$i]}', + '${repos_types[$i]}', + '', + '--- {}')" +done +