From a642e32110085f1c5d3a90c55ea0193aa851a5f9 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Wed, 23 Oct 2019 17:02:08 +0200 Subject: [PATCH 01/15] Make circleCI exit with error if sphinx warnings are generated. --- build_tools/circle/build_doc.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index c67278f4a5b85..9e96bb59ae2e9 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -203,5 +203,13 @@ then echo "$warnings" | sed 's/\/home\/circleci\/project\//
  • /g' echo '' ) > 'doc/_build/html/stable/_changed.html' + + check = `echo $warnings | grep "no warnings"` + if [ -z "$check" ] + then + echo "There are Sphinx Warnings in the documentation!" + echo "Please check doc/_build/html/stable/_changed.html" + exit 1 + fi fi From 96cf7502e5b0de8f3cd41363b260ba85888488e0 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Thu, 24 Oct 2019 17:07:32 +0200 Subject: [PATCH 02/15] Fix unsafe exit. --- build_tools/circle/build_doc.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 9e96bb59ae2e9..aba2bce532698 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -189,6 +189,8 @@ then if [ -z "$warnings" ] then warnings="/home/circleci/project/ no warnings" + else + check=1 fi echo "$warnings" @@ -204,8 +206,7 @@ then echo '' ) > 'doc/_build/html/stable/_changed.html' - check = `echo $warnings | grep "no warnings"` - if [ -z "$check" ] + if [ $check ] then echo "There are Sphinx Warnings in the documentation!" echo "Please check doc/_build/html/stable/_changed.html" From c19c4e3d936e60c91e9283c65ce23517bcd36fe2 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Thu, 24 Oct 2019 17:07:32 +0200 Subject: [PATCH 03/15] [doc build] Fix unsafe exit. --- build_tools/circle/build_doc.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 9e96bb59ae2e9..aba2bce532698 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -189,6 +189,8 @@ then if [ -z "$warnings" ] then warnings="/home/circleci/project/ no warnings" + else + check=1 fi echo "$warnings" @@ -204,8 +206,7 @@ then echo '' ) > 'doc/_build/html/stable/_changed.html' - check = `echo $warnings | grep "no warnings"` - if [ -z "$check" ] + if [ $check ] then echo "There are Sphinx Warnings in the documentation!" echo "Please check doc/_build/html/stable/_changed.html" From 4d07039ec9bb6f37aefd5e0d40fcdd5c26a194c2 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Wed, 6 Nov 2019 14:03:11 +0100 Subject: [PATCH 04/15] Look for example images in modified files. --- build_tools/circle/build_doc.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index aba2bce532698..f1000722637e0 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -58,6 +58,16 @@ get_build_type() { return fi changed_examples=$(echo "$filenames" | grep -E "^examples/(.*/)*plot_") + if [ -n "$filenames" ] + then + for af in ${filenames[@]} + do + page_figure=$(grep figure $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') + page_image=$(grep image $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') + done + fi + changed_examples+=$(echo ${page_figure::-1} | sed 's/_ /|/g') + changed_examples+=$(echo ${page_image::-1} | sed 's/_ /|/g') if [[ -n "$changed_examples" ]] then echo BUILD: detected examples/ filename modified in $git_range: $changed_examples From 0196e9e1d1b79ea7f9fbd2191f7791160ed28334 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Wed, 6 Nov 2019 14:15:39 +0100 Subject: [PATCH 05/15] Fix for no images in modified files. --- build_tools/circle/build_doc.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index f1000722637e0..25354f3df11e9 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -66,8 +66,14 @@ get_build_type() { page_image=$(grep image $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') done fi - changed_examples+=$(echo ${page_figure::-1} | sed 's/_ /|/g') - changed_examples+=$(echo ${page_image::-1} | sed 's/_ /|/g') + if [ -n "$page_figure" ] + then + changed_examples+=$(echo ${page_figure::-1} | sed 's/_ /|/g') + fi + if [ -n "$page_image" ] + then + changed_examples+=$(echo ${page_image::-1} | sed 's/_ /|/g') + fi if [[ -n "$changed_examples" ]] then echo BUILD: detected examples/ filename modified in $git_range: $changed_examples From c6735d9f722fb8fbb4ee1c70d1d7cef32e9a751a Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Wed, 6 Nov 2019 17:24:52 +0100 Subject: [PATCH 06/15] Exclude build_doc.sh --- build_tools/circle/build_doc.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 25354f3df11e9..9fdb9c843b217 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -62,8 +62,11 @@ get_build_type() { then for af in ${filenames[@]} do - page_figure=$(grep figure $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') - page_image=$(grep image $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') + if [ $af != "build_tools/circle/build_doc.sh" ] + then + page_figure=$(grep figure $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') + page_image=$(grep image $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') + fi done fi if [ -n "$page_figure" ] From a2f325558e1f828342c80c56f91d23b7f4ea590f Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Wed, 13 Nov 2019 00:37:07 +0100 Subject: [PATCH 07/15] Generalized implementation. Fix concatenation issues. --- build_tools/circle/build_doc.sh | 53 ++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 9e3ba2cf14be8..b94ea82c64998 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -58,32 +58,45 @@ get_build_type() { return fi changed_examples=$(echo "$filenames" | grep -E "^examples/(.*/)*plot_") - if [ -n "$filenames" ] - then - for af in ${filenames[@]} - do - if [ $af != "build_tools/circle/build_doc.sh" ] - then - page_figure=$(grep figure $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') - page_image=$(grep image $af | grep auto_example | awk -F "/" '{print $NF}' | sed 's/sphx_glr_//' | awk -F "_" '{OFS="_";$NF=""; print $0}') - fi - done - fi - if [ -n "$page_figure" ] - then - changed_examples+=$(echo ${page_figure::-1} | sed 's/_ /|/g') - fi - if [ -n "$page_image" ] + + if [[ -n "$filenames" ]] then - changed_examples+=$(echo ${page_image::-1} | sed 's/_ /|/g') + # get rst files + rst_files="$(echo "$filenames" | grep -E "rst$")" + + # get lines with figure or images + img_fig_lines="$(echo "$rst_files" | xargs grep -shE "(figure|image)::")" + + # get only auto_examples + auto_example_files="$(echo "$img_fig_lines" | grep auto_examples | awk -F "/" '{print $NF}')" + + # remove "sphx_glr_" from path and accept replace _\d\d\d.png with .py + image_paths="$(echo "$auto_example_files" | sed 's/sphx_glr_//' | sed -e 's/_[[:digit:]][[:digit:]][[:digit:]].png/.py/')" + + # get unique values + examples_in_rst="$(echo "$image_paths" | uniq | paste -sd '|')" fi + if [[ -n "$changed_examples" ]] then - echo BUILD: detected examples/ filename modified in $git_range: $changed_examples - pattern=$(echo "$changed_examples" | paste -sd '|') + if [[ -n "$examples_in_rst" ]] + then + pattern=$(echo "$changed_examples" | paste -sd '|')"|"$examples_in_rst + else + pattern=$(echo "$changed_examples" | paste -sd '|') + fi # pattern for examples to run is the last line of output - echo "$pattern" + echo BUILD: detected examples/ filename modified in $git_range: $pattern + echo $pattern return + else + if [[ -n "$examples_in_rst" ]] + then + # pattern for examples to run is the last line of output + echo BUILD: detected examples/ filename modified in $git_range: $pattern + echo $pattern + return + fi fi echo QUICK BUILD: no examples/ filename modified in $git_range: echo "$filenames" From bc0841548b10d34bf1bf3a5963b83cbabc055fd4 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Thu, 14 Nov 2019 15:46:32 +0100 Subject: [PATCH 08/15] Address comments --- build_tools/circle/build_doc.sh | 45 ++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index b94ea82c64998..7694f43415bd2 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -59,6 +59,14 @@ get_build_type() { fi changed_examples=$(echo "$filenames" | grep -E "^examples/(.*/)*plot_") + # The following is used to extract the list of filenames of example python + # files that sphinx-gallery needs to run to generate png files used as + # figures or images in the .rst files from the documentation. + # If the contributor changes a .rst file in a PR we need to run all + # the examples mentioned in that file to get sphinx build the + # documentation without generating spurious warnings related to missing + # png files. + if [[ -n "$filenames" ]] then # get rst files @@ -71,33 +79,34 @@ get_build_type() { auto_example_files="$(echo "$img_fig_lines" | grep auto_examples | awk -F "/" '{print $NF}')" # remove "sphx_glr_" from path and accept replace _\d\d\d.png with .py - image_paths="$(echo "$auto_example_files" | sed 's/sphx_glr_//' | sed -e 's/_[[:digit:]][[:digit:]][[:digit:]].png/.py/')" + scripts_names="$(echo "$auto_example_files" | sed 's/sphx_glr_//' | sed -e 's/_[[:digit:]][[:digit:]][[:digit:]].png/.py/')" # get unique values - examples_in_rst="$(echo "$image_paths" | uniq | paste -sd '|')" + examples_in_rst="$(echo "$scripts_names" | uniq )" fi + pattern="" + if [[ -n "$examples_in_rst" ]] + then + pattern=$($pattern)$(echo "$examples_in_rst" | paste -sd '|' ) + fi if [[ -n "$changed_examples" ]] then - if [[ -n "$examples_in_rst" ]] - then - pattern=$(echo "$changed_examples" | paste -sd '|')"|"$examples_in_rst - else - pattern=$(echo "$changed_examples" | paste -sd '|') - fi - # pattern for examples to run is the last line of output + pattern=$(echo $pattern)"|"$(echo "$changed_examples" | paste -sd '|') + fi + if [[ $pattern="|" ]] + then + pattern=$(echo "$pattern" | cut -c 2-) + fi + + # pattern for examples to run is the last line of output + if [[ -n "$pattern" ]] + then echo BUILD: detected examples/ filename modified in $git_range: $pattern echo $pattern return - else - if [[ -n "$examples_in_rst" ]] - then - # pattern for examples to run is the last line of output - echo BUILD: detected examples/ filename modified in $git_range: $pattern - echo $pattern - return - fi fi + echo QUICK BUILD: no examples/ filename modified in $git_range: echo "$filenames" } @@ -241,7 +250,7 @@ then if [ $check ] then - echo "There are Sphinx Warnings in the documentation!" + echo "Sphinx generated warnings when building the documentation related to files modified in this PR." echo "Please check doc/_build/html/stable/_changed.html" exit 1 fi From f91903ef284e99d972d4df5d640a2a7dee73631c Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Thu, 14 Nov 2019 15:51:42 +0100 Subject: [PATCH 09/15] Add comments. --- build_tools/circle/build_doc.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 7694f43415bd2..630d27f445f9f 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -85,15 +85,22 @@ get_build_type() { examples_in_rst="$(echo "$scripts_names" | uniq )" fi + #initialize pattern pattern="" + + # executed only if there are examples in the modified rst files if [[ -n "$examples_in_rst" ]] then pattern=$($pattern)$(echo "$examples_in_rst" | paste -sd '|' ) fi + + # executed only if some examples have been modified if [[ -n "$changed_examples" ]] then pattern=$(echo $pattern)"|"$(echo "$changed_examples" | paste -sd '|') fi + + # cut trailing initial "|" if only examples in rst need to be built if [[ $pattern="|" ]] then pattern=$(echo "$pattern" | cut -c 2-) From 2fef5e77af9c94cebb6c19f13f76c33f87cc94df Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Fri, 15 Nov 2019 11:19:10 +0100 Subject: [PATCH 10/15] Implement @thomasjpfan solution --- build_tools/circle/build_doc.sh | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 630d27f445f9f..29856291a7e3d 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -85,32 +85,23 @@ get_build_type() { examples_in_rst="$(echo "$scripts_names" | uniq )" fi - #initialize pattern - pattern="" - # executed only if there are examples in the modified rst files if [[ -n "$examples_in_rst" ]] then - pattern=$($pattern)$(echo "$examples_in_rst" | paste -sd '|' ) + if [[ -n "$changed_examples" ]] + then + changed_examples="$changed_examples|$examples_in_rst" + else + changed_examples="$examples_in_rst" + fi fi - # executed only if some examples have been modified if [[ -n "$changed_examples" ]] then - pattern=$(echo $pattern)"|"$(echo "$changed_examples" | paste -sd '|') - fi - - # cut trailing initial "|" if only examples in rst need to be built - if [[ $pattern="|" ]] - then - pattern=$(echo "$pattern" | cut -c 2-) - fi - - # pattern for examples to run is the last line of output - if [[ -n "$pattern" ]] - then - echo BUILD: detected examples/ filename modified in $git_range: $pattern - echo $pattern + echo BUILD: detected examples/ filename modified in $git_range: $changed_examples + pattern=$(echo "$changed_examples" | paste -sd '|') + # pattern for examples to run is the last line of output + echo "$pattern" return fi From fa3ca3fc0d4bb65c4a0bf334e9422ab67233858f Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Fri, 15 Nov 2019 07:05:57 -0800 Subject: [PATCH 11/15] CLN Uses warnings directly --- build_tools/circle/build_doc.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 29856291a7e3d..15f3632d103d7 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -229,8 +229,6 @@ then if [ -z "$warnings" ] then warnings="/home/circleci/project/ no warnings" - else - check=1 fi echo "$warnings" @@ -246,7 +244,7 @@ then echo '' ) > 'doc/_build/html/stable/_changed.html' - if [ $check ] + if [ "$warnings" != "/home/circleci/project/ no warnings" ] then echo "Sphinx generated warnings when building the documentation related to files modified in this PR." echo "Please check doc/_build/html/stable/_changed.html" From 8924e17df96d5504b69e8e6becbd23ff460bf8a5 Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Fri, 15 Nov 2019 07:09:13 -0800 Subject: [PATCH 12/15] TST Introduces error --- doc/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install.rst b/doc/install.rst index d45e9f3367ff1..1365bbdc060c3 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -1,8 +1,8 @@ .. _installation-instructions: -======================= +====================== Installing scikit-learn -======================= +====================== There are different ways to install scikit-learn: From f6de0a0b4ef104c939f02bd03a24eb26f0724b63 Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Fri, 15 Nov 2019 07:50:13 -0800 Subject: [PATCH 13/15] TST Fixes error --- doc/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install.rst b/doc/install.rst index 1365bbdc060c3..d45e9f3367ff1 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -1,8 +1,8 @@ .. _installation-instructions: -====================== +======================= Installing scikit-learn -====================== +======================= There are different ways to install scikit-learn: From df8adcacf178fb34d1f012ff19e7ab024c9bb79a Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Fri, 15 Nov 2019 08:39:12 -0800 Subject: [PATCH 14/15] STY Changes --- build_tools/circle/build_doc.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 15f3632d103d7..a45ca96131f77 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -88,12 +88,12 @@ get_build_type() { # executed only if there are examples in the modified rst files if [[ -n "$examples_in_rst" ]] then - if [[ -n "$changed_examples" ]] - then - changed_examples="$changed_examples|$examples_in_rst" - else - changed_examples="$examples_in_rst" - fi + if [[ -n "$changed_examples" ]] + then + changed_examples="$changed_examples|$examples_in_rst" + else + changed_examples="$examples_in_rst" + fi fi if [[ -n "$changed_examples" ]] @@ -104,7 +104,6 @@ get_build_type() { echo "$pattern" return fi - echo QUICK BUILD: no examples/ filename modified in $git_range: echo "$filenames" } From 5be4023147cf3ed2dd4bd94eec841726101c23e2 Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Sat, 16 Nov 2019 15:11:17 -0800 Subject: [PATCH 15/15] ENH Also grep for thumb pngs --- build_tools/circle/build_doc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index a45ca96131f77..c3481bd6c58c1 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -78,8 +78,8 @@ get_build_type() { # get only auto_examples auto_example_files="$(echo "$img_fig_lines" | grep auto_examples | awk -F "/" '{print $NF}')" - # remove "sphx_glr_" from path and accept replace _\d\d\d.png with .py - scripts_names="$(echo "$auto_example_files" | sed 's/sphx_glr_//' | sed -e 's/_[[:digit:]][[:digit:]][[:digit:]].png/.py/')" + # remove "sphx_glr_" from path and accept replace _(\d\d\d|thumb).png with .py + scripts_names="$(echo "$auto_example_files" | sed 's/sphx_glr_//' | sed -e 's/_([[:digit:]][[:digit:]][[:digit:]]|thumb).png/.py/')" # get unique values examples_in_rst="$(echo "$scripts_names" | uniq )"