001package Torello.HTML.Tools.Images;
002
003import Torello.HTML.*;
004import Torello.Java.*;
005
006import Torello.HTML.NodeSearch.TagNodeFind;
007import Torello.Java.Additional.Ret2;
008import Torello.Java.Additional.Counter;
009import Torello.Java.Shell.C;
010
011import java.net.*;
012import java.io.*;
013import java.util.*;
014import java.util.regex.*;
015import java.util.function.*;
016import javax.imageio.*;
017import java.util.concurrent.*;
018import java.util.concurrent.locks.*;
019
020import java.awt.image.BufferedImage;
021
022/**
023 * <CODE>ImageScraper - Documentation.</CODE><BR /><BR />
024 * <EMBED CLASS="external-html" DATA-FILE-ID="ISR">
025 */
026public class ImageScraper
027{
028    /**
029     * This is the default maximum wait time for an image to download ({@value}).  This value may
030     * be reset or modified by instantiating a {@code ImageScraper.AdditionalParameters} class, and
031     * passing the desired values to the constructor.  This value is measured in units of
032     * {@code public static final java.util.concurrent.TimeUnit MAX_WAIT_TIME_UNIT}
033     *
034     * @see #MAX_WAIT_TIME_UNIT
035     */
036    public static final long        MAX_WAIT_TIME       = 10;
037
038    /**
039     * This is the default measuring unit for the {@code static final long MAX_WAIT_TIME} member.
040     * This value may be reset or modified by instantiating a 
041     * {@code ImageScraper.AdditionalParameters} class, and passing the desired values to the
042     * constructor.
043     *
044     * @see #MAX_WAIT_TIME
045     */
046    public static final TimeUnit MAX_WAIT_TIME_UNIT  = TimeUnit.SECONDS;
047
048    /** <EMBED CLASS="external-html" DATA-FILE-ID="ISUA"> */
049    public static String USER_AGENT = "Chrome/61.0.3163.100";
050
051    private final   Iterable<String>            source;
052    private final   URL                         originalPageURL;
053
054    private final   String                      targetDirectory;
055    private final   TargetDirectoryRetriever    targetDirectoryRetriever;
056    private final   ImageReceiver               imageReceiver;
057
058    /**
059     * <CODE>TargetDirectoryRetriever - Documentation.</CODE><BR /><BR />
060     * <EMBED CLASS="external-html" DATA-FILE-ID="TDR">
061     */
062    @FunctionalInterface
063    public static interface TargetDirectoryRetriever extends java.io.Serializable
064    {
065        /** <EMBED CLASS="external-html" DATA-FILE-ID="SVUIDFI">  */
066        public static final long serialVersionUID = 1;
067
068        /**
069         * The {@code dir(...)} method within this interface will be called each time that an image
070         * successfully downloads from the internet.  It's purpose is to allow the programmer to
071         * supply a target directory for where to store this downloaded image.  Implement the
072         * lone-method from this interface, and images will be saved to individual save-directories
073         * on an image-by-image basis.  If a {@code interface TargetDirectorieRetriever} is not
074         * provided, then all images will be saved to a single target-directory (or the
075         * {@code interface 'ImageReceiver"} must be implemented).
076         *
077         * @param url This is the {@code URL} that was used to connect to the internet, and
078         * download the image in question.
079         *
080         * @param fileName This parameter will receive the computed filename of the image.
081         *
082         * @param imageFormat This identifies whether the image-in-question is a {@code GIF, JPG, 
083         * PNG} etc...  Remember the image might not be saved by the same name which was used in 
084         * the HTML on the website from which this was downloaded.
085         *
086         * @param iteratorCount <EMBED CLASS="external-html" DATA-FILE-ID="ISIC">
087         *
088         * @param successCount <EMBED CLASS="external-html" DATA-FILE-ID="ISSC">
089         *
090         * @return It is up to the user implement this method such that it returns a {@code String}
091         * that identifies an appropriate directory in the local filesystem where the image may be
092         * saved.
093         */
094        public String dir
095            (URL url, String fileName, IF imageFormat, int iteratorCount, int successCount);
096    }
097
098    /**
099     * <CODE>ImageReceiver - Documentation.</CODE><BR /><BR />
100     * <EMBED CLASS="external-html" DATA-FILE-ID="IMGR">
101     */
102    @FunctionalInterface
103    public static interface ImageReceiver extends java.io.Serializable
104    {
105        /** <EMBED CLASS="external-html" DATA-FILE-ID="SVUIDFI">  */
106        public static final long serialVersionUID = 1;
107
108        /**
109         * Implement this class if saving image files to a target-directory on the file-system is
110         * not acceptable, and the programmer wishes to do something else with the downloaded
111         * images.  The lone-method in this interface (the "save" method) will be invoked each
112         * time and image is downloaded.
113         *
114         * @param url This is the {@code URL} that was used to connect to the internet, and
115         * download the image in question.
116         *
117         * @param fileName This parameter will receive the computed filename of the image.
118         *
119         * @param imageFormat This identifies whether the image-in-question is a {@code GIF, JPG,
120         * PNG, etc...}  Remember the image might not be saved by the same name which was used in
121         * the HTML on the website from which this was downloaded.
122         *
123         * @param iteratorCount <EMBED CLASS="external-html" DATA-FILE-ID="ISIC">
124         *
125         * @param successCount <EMBED CLASS="external-html" DATA-FILE-ID="ISSC">
126         *
127         * @param image This is the newly downloaded image.
128         */
129        public void save(
130            URL url, String fileName, IF imageFormat, int iteratorCount,
131            int successCount, BufferedImage image
132        );
133    }
134
135    /**
136     * <CODE>FileNameRetriever - Documentation.</CODE><BR /><BR />
137     * <EMBED CLASS="external-html" DATA-FILE-ID="FNR">
138     */
139    @FunctionalInterface
140    public static interface FileNameRetriever extends java.io.Serializable
141    {
142        /** <EMBED CLASS="external-html" DATA-FILE-ID="SVUIDFI">  */
143        public static final long serialVersionUID = 1;
144
145        /**
146         * The intention of implementing this method is to provide the code with an 'adjusted'
147         * file name for saving images downloaded from the internet.
148         *
149         * @param url This is the {@code URL} that was used to connect to the internet, and
150         * download the image in question.
151         *
152         * @param imageFormat This identifies whether the image-in-question is a {@code GIF, JPG, 
153         * PNG,} etc...  Remember the image might not be saved by the same name which was used in
154         * the HTML on the website from which this was downloaded.
155         *
156         * @param iteratorCount <EMBED CLASS="external-html" DATA-FILE-ID="ISIC">
157         * @param successCount <EMBED CLASS="external-html" DATA-FILE-ID="ISSC">
158         *
159         * @return utilizing the information provided in the method-signature, the programmer is
160         * expected to provide a file-name for saving the image that was provided.
161         */
162        public String fileName(URL url, IF imageFormat, int iteratorCount, int successCount);
163    }
164
165    // *************************************************************************************
166    // source is Iterable<URL>
167    // *************************************************************************************
168
169    private static final Iterable<String> URLVecToStringVec(Iterable<URL> source)
170    {
171        Vector<String> ret = new Vector<>();
172        source.forEach((URL url) -> ret.add(url.toString()));
173        return ret;
174    }
175
176    /**
177     * Convenience Constructor.  Invokes {@link #ImageScraper(URL, Iterable, String)}
178     * <BR /><BR />Converts {@code Iterable<URL>} to {@code Iterable<String>}.
179     */
180    public ImageScraper(Iterable<URL> source, String targetDirectory)
181    { this(null, URLVecToStringVec(source), targetDirectory); }
182
183    /**
184     * Convenience Constructor.  Invokes {@link #ImageScraper(URL, Iterable, TargetDirectoryRetriever)}
185     * <BR /><BR />Converts {@code Iterable<URL>} to {@code Iterable<String>}.
186     */
187    public ImageScraper(Iterable<URL> source, TargetDirectoryRetriever targetDirectoryRetriever)
188    { this(null, URLVecToStringVec(source), targetDirectoryRetriever); }
189
190    /**
191     * Convenience Constructor.  Invokes {@link #ImageScraper(URL, Iterable, ImageReceiver)}
192     * <BR /><BR />Converts {@code Iterable<URL>} to {@code Iterable<String>}.
193     */
194    public ImageScraper(Iterable<URL> source, ImageReceiver imageReceiver)
195    { this(null, URLVecToStringVec(source), imageReceiver); }
196
197    // *************************************************************************************
198    // source is Iterable<TagNode>, URL
199    // *************************************************************************************
200
201    private static final Iterable<String> TagNodeVecToStringVec(Iterable<TagNode> source)
202    {
203        Vector<String> ret = new Vector<>();
204        source.forEach((TagNode tn) -> ret.add(tn.AV("src")));
205        return ret;
206    }
207
208    /**
209     * Convenience Constructor.  Invokes {@link #ImageScraper(URL, Iterable, String)}
210     * <BR /><BR />Converts {@code Iterable<TagNode>} to {@String[]} using {@link TagNode#AV(String)}
211     * 
212     * @param source This may be any java {@code Iterable<TagNode>}.  The {@code TagNode's} are
213     * expected to contain HTML {@code <IMG SRC="...">} tags.
214     */
215    public ImageScraper(Iterable<TagNode> source, URL originalPageURL, String targetDirectory)
216    { this(originalPageURL, TagNodeVecToStringVec(source), targetDirectory); }
217
218    /**
219     * Convenience Constructor.  Invokes {@link #ImageScraper(URL, Iterable, TargetDirectoryRetriever)}
220     * <BR /><BR />Converts {@code Iterable<TagNode>} to {@String[]} using {@link TagNode#AV(String)}
221     * 
222     * @param source This may be any java {@code Iterable<TagNode>}.  The {@code TagNode's} are
223     * expected to contain HTML {@code <IMG SRC="...">} tags.
224     */
225    public ImageScraper(Iterable<TagNode> source, URL originalPageURL, TargetDirectoryRetriever targetDirectoryRetriever)
226    { this(originalPageURL, TagNodeVecToStringVec(source), targetDirectoryRetriever); }
227
228    /**
229     * Convenience Constructor.  Invokes {@link #ImageScraper(URL, Iterable, ImageReceiver)}
230     * <BR /><BR />Converts {@code Iterable<TagNode>} to {@String[]} using {@link TagNode#AV(String)}
231     * 
232     * @param source This may be any java {@code Iterable<TagNode>}.  The {@code TagNode's} are
233     * expected to contain HTML {@code <IMG SRC="...">} tags.
234     */
235    public ImageScraper(Iterable<TagNode> source, URL originalPageURL, ImageReceiver imageReceiver)
236    { this(originalPageURL, TagNodeVecToStringVec(source), imageReceiver); }
237
238    // *************************************************************************************
239    // source is Iterable<String>, URL
240    // *************************************************************************************
241
242    /**
243     * Constructor that allows a user to provide a set of {@code URL's} as {@code String's} to the
244     * download mechanism.
245     *
246     * @param source This is a {@code Vector<String>} of Image {@code URL's} saved as a
247     * {@code String}.
248     *
249     * @param originalPageURL <EMBED CLASS="external-html" DATA-FILE-ID="ISOPURL">
250     *
251     * @param targetDirectory When this constructor is used, this {@code String} parameter
252     * identifies the directory to where files must be saved.
253     *
254     * @throws NullPointerException If any of the elements of the input {@code Iterable<String>}
255     * are null elements, then this Exception shall be thrown.
256     *
257     * @throws WritableDirectoryException This constructor shall check that parameter
258     * {@code 'targetDirectory'} exists on the file-system, and is writable.  A small, temporary,
259     * file shall be written to check this.
260     */
261    public ImageScraper(URL originalPageURL, Iterable<String> source, String targetDirectory)
262    {
263        this.source                     = source;
264        this.originalPageURL            = originalPageURL;
265
266        // Ensures that the target directory exists, and is writable
267        WritableDirectoryException.check(targetDirectory);
268
269        // Makes sure that the directory ends with a slash / file-separator.
270        if (! targetDirectory.endsWith(File.separator)) if (targetDirectory.length() > 0)
271            targetDirectory = targetDirectory + File.separator;
272
273        this.targetDirectory            = targetDirectory;
274        this.targetDirectoryRetriever   = null;
275        this.imageReceiver              = null;
276
277        if (source == null)
278            throw new NullPointerException("parameter source is null");
279
280        if (targetDirectory == null)
281            throw new NullPointerException("parameter targetDirectory is null");
282    }
283
284    /**
285     * Constructor that allows a user to provide a set of {@code URL's} as {@code String's} to
286     * the download mechanism.
287     *
288     * @param source This is a {@code Vector<String>} of Image {@code URL's} saved as a
289     * {@code String}.
290     *
291     * @param originalPageURL <EMBED CLASS="external-html" DATA-FILE-ID="ISOPURL">
292     *
293     * @param targetDirectoryRetriever This parameter must implement the static-inner
294     * {@code class TargetDirectoryRetriever}.  This parameter allows the programmer to make a 
295     * decision where image-files are stored after they are downloaded one a file-by-file basis.
296     *
297     * @throws NullPointerException If any of the elements of the input {@code Iterable<String>}
298     * are null elements, then this Exception shall be thrown.
299     */
300    public ImageScraper(
301        URL originalPageURL, Iterable<String> source,
302        TargetDirectoryRetriever targetDirectoryRetriever
303    )
304    {
305        this.source                     = source;
306        this.originalPageURL            = originalPageURL;
307        this.targetDirectory            = null;
308        this.targetDirectoryRetriever   = targetDirectoryRetriever;
309        this.imageReceiver              = null;
310
311        if (source == null)
312            throw new NullPointerException("parameter source is null");
313
314        if (targetDirectoryRetriever == null)
315            throw new NullPointerException("targetDirectoryRetriever is null");
316    }
317
318    /**
319     * Constructor that allows a user to provide a set of {@code URL's} as {@code String's} to the
320     * download mechanism.
321     *
322     * @param source This is a {@code Vector<String>} of Image {@code URL's} saved as a
323     * {@code String}.
324     *
325     * @param originalPageURL <EMBED CLASS="external-html" DATA-FILE-ID="ISOPURL">
326     *
327     * @param imageReceiver This parameter allows the programmer to circumvent the "save-to-file"
328     * portion of the code, and instead send the downloaded image to this interface.
329     *
330     * @throws NullPointerException If any of the elements of the input {@code Iterable<String>} 
331     * are null elements, then this exception shall be thrown.
332     */
333    public ImageScraper(URL originalPageURL, Iterable<String> source, ImageReceiver imageReceiver)
334    {
335        this.source                     = source;
336        this.originalPageURL            = originalPageURL;
337
338        this.targetDirectory            = null;
339        this.targetDirectoryRetriever   = null;
340        this.imageReceiver              = imageReceiver;
341
342        if (source == null)
343            throw new NullPointerException("parameter source is null");
344
345        if (imageReceiver == null)
346            throw new NullPointerException("imageReceiver is null");
347    }
348
349    // *************************************************************************************
350    // *************************************************************************************
351    // More available download configuration parameters
352    // *************************************************************************************
353    // *************************************************************************************
354
355    /**
356     * <CODE>AdditionalParameters - Documentation.</CODE><BR /><BR />
357     * <EMBED CLASS="external-html" DATA-FILE-ID="ADPA">
358     */
359    public static class AdditionalParameters
360    {
361        /** <EMBED CLASS="external-html" DATA-FILE-ID="SVUID"> */
362        public static final long serialVersionUID = 1;
363    
364        /**
365         * When this field is <B>TRUE</B>, if an attempt to download an image generates an
366         * exception, the exception-throw <I>will not halt the download</I>, but rather the image
367         * will be skipped, and download attempt will be performed on the next image in the list.
368         * The exception will be stored in the {@code 'Results'} return object.
369         */
370        public final boolean                skipOnIOException;
371
372        /**
373         * When this field is null, it is ignored.  If this field is not null, then before any
374         * {@code URL} is connected for download, the downloaded mechanism will ask this
375         * {@code URL-Predicate} for permission first.  If this {@code Predicate} returns
376         * <B>FALSE</B> for a particular <B>URL,</B> that image will not be downloaded, and
377         * instead, skipped.
378         */
379        public final Predicate<URL>         skipURL;
380
381        /**
382         * When this field is null, it is ignored.  If not null, this {@code String} will be
383         * <I>prepended</I> to each file-name that is saved or stored to the file-system.
384         */
385        public final String                 fileNamePrefix;
386
387        /**
388         * When true, images will be saved according to a counter.  When this is <B>FALSE</B>, the
389         * software will attempt to save these images using their original filenames - picked from
390         * the <B>URL.</B> Saving using a counter is the default behaviour for this class.
391         */
392        public final boolean                useDefaultCounterForImageFileNames;
393
394        /**
395         * When this field is null, it is ignored.  If not null, each time an image is written to
396         * the file-system, this {@code java.util.function.Function<URL, String>} will be queried
397         * for a file-name before writing the the image-file to the file-system.  If this field is
398         * non-null, but images are being sent to {@code Consumer<BufferedImage, IF> 
399         * downloadedImageAltTarget}, rather than being saved to the file-system, then this field
400         * is <I>also ignored</I>.
401         */
402        public final FileNameRetriever      getImageFileSaveName;
403
404        /**
405         * This scraper has the ability to decode and save {@code Base-64} Images.  If an
406         * {@code Iterable<TagNode>} is passed to the constructor, and one of those
407         * {@code TagNode's} contain an Image Element
408         * ({@code <IMG SRC="data:image/jpeg;base64,...data">}) this class has the ability to
409         * interpret and save the image to a regular image file.  By default, {@code Base-64}
410         * images are skipped, but they can also be downloaded as well.
411         */
412        public final boolean                skipBase64EncodedImages;
413
414        /**
415         * If you do not want the downloader to hang on an image, which is sometimes an issue
416         * depending upon the site from which you are making a request, set this parameter, and the
417         * downloader will not wait past that amount of time to download an image.  The default
418         * value for this parameter is {@code 10 seconds}.  If you do not wish to set the
419         * max-wait-time "the download time-out" counter, then leave the parameter
420         * {@code "waitTimeUnits"} set to {@code null}, and this parameter will be ignored.
421         */
422        public final long                   maxDownloadWaitTime;
423
424        /**
425         * This is the "unit of measurement" for the field {@code long maxDownloadWaitTime}.
426         * <BR /><BR /><B>NOTE:</B> <I>This parameter may be {@code null}, and if it is
427         * <SPAN STYLE="color: red;"> both <B>this</B> parameter and the parameter <B>{@code long
428         * maxDownloadWaitTime}</B> will be ignored</SPAN></I>, and the default maximum-wait-time
429         * (download time-out settings) will be used instead.
430         *
431         * <BR /><BR /><B>READ:</B> java.util.concurrent.*; package, and about the {@code class
432         * java.util.concurrent.TimeUnit} for more information.
433         */
434        public final TimeUnit               waitTimeUnits;
435
436        /**
437         * Use this constructor to instantiate this class.  Read what each of these parameters
438         * means to the downloader, by reading the comment information for each of these fields
439         * in this class (above).
440         *
441         * @param skipOnIOException This will "skip" an image, and prevent the downloading process from
442         * halting if an image fails to download
443         *
444         * @param skipURL A java {@code Predicate} for deciding which images should be skipped.
445         * This parameter may be 'null.'  If it is, it will be ignored, and the downloader will
446         * attempt to download all images.
447         *
448         * @param fileNamePrefix A standard Java-{@code String} may be inserted before the
449         * file-name of each image downloaded, as a 'file-name prefix'.  This parameter may be
450         * null, and if it is file-name prefixes will not be used.
451         *
452         * @param useDefaultCounterForImageFileNames It is usually a good idea to replace the
453         * file-name for an image retrieved from a web-site with a simple, three-digit,
454         * counter-name.  Image file names on a web-site can often be long {@code PKID Strings}
455         * obtained from {@code SQL} database queries. To use a standard "counter" set this
456         * parameter to <B>TRUE</B>.
457         *
458         * @param getImageFileSaveName This parameter may be used to convert image file-names used
459         * on a web-page to user-generated image-file-names.  This parameter may be null, and if it
460         * is - it will be ignored.  If this parameter is non-null, it takes precedence over the
461         * {@code boolean} passed to parameter {@code 'useDefaultCounterForImageFileNames'}
462         *
463         * @param skipBase64EncodedImages This will order the downloader to convert and save HTML
464         * Image Elements whose image-data was encoded into HTML Element, itself, using
465         * {@code Base-64} Image-Encoding.  Thumbnails and other small images are sometimes stored
466         * on web-pages using such encoding.
467         *
468         * @param maxDownloadWaitTime This parameter will be ignored unless a non-null value has
469         * been passed to parameter {@code 'waitTimeUnits'}.  This may be used to prevent the
470         * downloader from hanging when collecting images for a web-page.
471         *
472         * @param waitTimeUnits This is java {@code class TimeUnit} parameter for describing what
473         * units are being used for the previous parameter, {@code 'maxDownloadWaitTime'}.
474         */
475        public AdditionalParameters(
476            boolean                 skipOnIOException,
477            Predicate<URL>          skipURL,
478            String                  fileNamePrefix,
479            boolean                 useDefaultCounterForImageFileNames,
480            FileNameRetriever       getImageFileSaveName,
481            boolean                 skipBase64EncodedImages,
482            long                    maxDownloadWaitTime,
483            TimeUnit                waitTimeUnits
484        )
485        {
486            this.skipOnIOException                      = skipOnIOException;
487            this.skipURL                                = skipURL;
488            this.fileNamePrefix                         = fileNamePrefix;
489            this.useDefaultCounterForImageFileNames     = useDefaultCounterForImageFileNames;
490            this.getImageFileSaveName                   = getImageFileSaveName;
491            this.skipBase64EncodedImages                = skipBase64EncodedImages;
492            this.maxDownloadWaitTime                    = maxDownloadWaitTime;
493            this.waitTimeUnits                          = waitTimeUnits;
494
495            if (maxDownloadWaitTime < 0) throw new IllegalArgumentException(
496                "You have passed a negative number for parameter maxDownloadWaitTime, and this is " +
497                "not allowed here."
498            );
499        }
500
501        /**
502         * This constructor will return an instance of {@code AdditionalParameters} whose values
503         * provide the following <B>MOST COMMON</B> behaviour choices:
504         *
505         * <BR /><TABLE CLASS="BRIEFTABLE">
506         * <TR><TH>Parameter</TH><TH>Value</TH></TR>
507         * <TR><TD>{@code skipOnIOException}</TD><TD>{@code TRUE}</TD></TR>
508         * <TR><TD>{@code useDefaultCounterForImageFileNames}</TD><TD>{@code TRUE}</TD></TR>
509         * <TR><TD>{@code skipBase64EncodedImages}</TD><TD>{@code FALSE}</TD></TR>
510         * <TR><TD COLSPAN="2"><I>All other parameters set to 'null', and will be ignored.</I>
511         * </TD></TR>
512         * </TABLE>
513         */
514        public AdditionalParameters()
515        { this(true, null, null, true, null, false, 0, null); }
516    }
517
518    // *************************************************************************************
519    // *************************************************************************************
520    // Results inner class
521    // *************************************************************************************
522    // *************************************************************************************
523
524    /**
525     * <CODE>ImageScraper Results - Documentation.</CODE><BR /><BR />
526     * <EMBED CLASS="external-html" DATA-FILE-ID="ISRES">
527     */
528    public static class Results
529    {
530        /**
531         * The java serializable tools can be very beneficial for saving the state of a program you
532         * are testing.  Though it is unlikely a programmer would want to transmit this
533         * 'results-report' class around (or at least I cannot think of much use), saving the state
534         * of web-page scrape and all the testing routines that have been used is something that
535         * can be really helpful.  <I><SPAN STYLE="color: red;">This is why most of the classes
536         * that can be created / instantiated - a.k.a. non-static classes - implement the
537         * Serializable interface</SPAN></I>.  It's a great debugging tool.
538         */
539        public static final long serialVersionUID = 1;
540
541        /**
542         * This will contain a complete list of the {@code URL's} that were retrieved (or generated-
543         * <I>if partially-resolved 'relative' {@code URL's} occurred</I>).  Every image downloaded
544         * (or attempted for download) will have its {@code URL} saved here.
545         *
546         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
547         * retrieval order.
548         */
549        public final URL[] urls;
550
551        /**
552         * If the "skip" {@code Predicate} declares that a particular image-download should not be
553         * attempted, <I>FALSE</I> will be stored in this array.
554         *
555         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
556         * retrieval order.
557         */
558        public final boolean[] skipped;
559
560        /**
561         * The names of the files that were retrieved and/or stored will be in this array.
562         * If this image were skipped or an exception occurred, the array position for that
563         * {@code URL} would contain 'null'.
564         *
565         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
566         * retrieval order.
567         */
568        public final String[] fileNames;
569
570        /**
571         * The location of the file-name saved directory, if an image did not successfully save to
572         * the file system, or if an {@code ImageReceiver} were used, then the array-location would
573         * contain {@code 'null.'}
574         *
575         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
576         * retrieval order.
577         */
578        public final String[] saveDirectories;
579
580        /**
581         * The image type of the files that were retrieved will be stored in this array.
582         *
583         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
584         * retrieval order.
585         */
586        public final IF[] imageFormats;
587
588        /**
589         * If an image download fails, this will contain a record of the exception.
590         *
591         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
592         * retrieval order.
593         *
594         * <BR /><BR />If the download succeeded, then the associated array location would contain
595         * 'null.'
596         */
597        public final Exception[] exceptions;
598
599        /**
600         * This will contain a list of long-integers, each of which will have the file-size of the
601         * downloaded image.  If the programmer has elected for the {@code 'ImageReceiver'} option
602         * - <I>rather than direct download of the images to the underlying file-system</I> (save to
603         * lambda, instead of save-as-file) - then the "fileSize" will be a calculated file-size,
604         * and not reflect the actual size of any file on the file-system.  Obviously, this is
605         * because no file was created!
606         *
607         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
608         * retrieval order.
609         */
610        public final long[] sizes;
611
612        /**
613         * This will contain a list of integers, each of which shall have the image-widths of the 
614         * downloaded images.
615         *
616         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
617         * retrieval order.
618         */
619        public final int[] widths;
620
621        /**
622         * This shall contain a list of integers, each of which shall have the image-heights of 
623         * the downloaded images.
624         *
625         * <BR /><BR />The index of this array will be parallel to the input-source {@code URL}
626         * retrieval order.
627         */
628        public final int[] heights;
629
630        /** next result received array position. */
631        int pos = 0;
632
633        /** number of successfully saved images. */
634        int successCounter = 0;
635
636        /** When images are downloaded, log information may be sent here */
637        Appendable log = null;
638
639        Results(int size, Appendable log)
640        {
641            this.log = log;
642
643            urls                = new URL[size];
644            skipped             = new boolean[size];
645            fileNames           = new String[size];
646            saveDirectories     = new String[size];
647            imageFormats        = new IF[size];
648            exceptions          = new Exception[size];
649            sizes               = new long[size];
650            widths              = new int[size];
651            heights             = new int[size];
652
653            for (int i=0; i < size; i++)
654            {
655                urls[i]             = null;
656                skipped[i]          = false;
657                fileNames[i]        = null;
658                saveDirectories[i]  = null;
659                imageFormats[i]     = null;
660                exceptions[i]       = null;
661                sizes[i]            = -1;
662                widths[i]           = -1;
663                heights[i]          = -1;
664            }
665        }
666
667        void nullURL() throws IOException // The Appendable throws this
668        {
669            if (log != null) log.append
670                ("\t\t" + C.RED + "No URL was passed, or URL not found." + C.RESET + '\n');
671
672            skipped[pos]            = true;
673            pos++;
674        }
675
676        void urlException(String src, Exception e) throws IOException // The Appendable throws this
677        {
678            if (log != null) log.append(
679                "\t\t" + C.RED + "Failed Instantiate URL, src = " + src + ", " +
680                e.getClass().getName() + ": " + e.getMessage() + C.RESET + '\n'
681            );
682
683            skipped[pos]            = true;
684            exceptions[pos]         = e;
685            pos++;
686        }
687    
688        void skippedURL(URL url) throws IOException // The Appendable throws this
689        {
690            if (log != null) log.append("\t\t" + C.YELLOW + "*** SKIPPING" + C.RESET + '\n');
691
692            urls[pos]               = url;
693            skipped[pos]            = true;
694            pos++;
695        }
696
697        void downloadException(URL url, Exception e) throws IOException // The Appendable throws this
698        {
699            String msg = (e.getMessage() != null) 
700                ? e.getMessage()
701                : "no exception-message provided, [e.getMessage()==null]";
702
703            if (log != null) log.append(
704                "\t\t" + C.RED + "DOWNLOAD-EXCEPTION:\t" + e.getClass().getName() + ": " + msg + C.RESET + '\n' +
705                "\t\t" + "While Downloading URL:\t" + url.toString() + '\n'
706            );
707
708            urls[pos]               = url;
709            skipped[pos]            = true;
710            exceptions[pos]         = e;
711            pos++;
712        }
713
714        void imageReceiverSuccess
715            (URL url, String fileName, IF ext, long size, int width, int height)
716            throws IOException // The Appendable throws this
717        {
718            if (log != null) log.append(
719                "\t\t" + C.YELLOW + "Successfully sent [" + fileName + '.' + ext.extension + "] to class ImageReceiver" + C.RESET + '\n'
720            );
721
722            urls[pos]               = url;
723            fileNames[pos]          = fileName + '.' + ext.extension;
724            imageFormats[pos]       = ext;
725            sizes[pos]              = size;
726            widths[pos]             = width;
727            heights[pos]            = height;
728            pos++;
729            successCounter++;
730        }
731
732        void saveSuccess(
733                URL url, String targetDirectory, String fileName, IF ext, long size,
734                int width, int height
735            )
736            throws IOException // The Appendable throws this
737        {
738            if (log != null) log.append(
739                "\t\t" + C.YELLOW + "File Saved:\t" + targetDirectory + fileName + "." +
740                ext.extension + C.RESET + '\n'
741            );
742
743            urls[pos]               = url;
744            saveDirectories[pos]    = targetDirectory;
745            fileNames[pos]          = fileName + '.' + ext.extension;
746            imageFormats[pos]       = ext;
747            sizes[pos]              = size;
748            widths[pos]             = width;
749            heights[pos]            = height;
750            pos++;
751            successCounter++;
752        }
753
754        void saveFail(URL url, String targetDirectory, String fileName, IF ext, Exception e)
755             throws IOException // The Appendable throws this
756        {
757            if (log != null) log.append(
758                "\t\t" + C.RED + "***FILE-SAVE-EXCEPTION:\t" + targetDirectory + fileName + "." +
759                ext.extension +
760                "\t\t" + e.getClass().getName() + ": " + e.getMessage() + C.RESET + '\n'
761            );
762
763            urls[pos]               = url;
764            skipped[pos]            = true;
765            saveDirectories[pos]    = targetDirectory;
766            fileNames[pos]          = fileName + '.' + ext.extension;
767            imageFormats[pos]       = ext;
768            exceptions[pos]         = e;
769            pos++;
770        }
771
772        void skipB64(String imgFormatStr, String encodedPartialStr)
773            throws IOException // The Appendable throws this
774        {
775            if (log != null) log.append(
776                "\t\t" + C.YELLOW + "Skipping B64 Encoded String: " + imgFormatStr + ", " + encodedPartialStr + C.RESET + '\n'
777            );
778
779            skipped[pos]            = true;
780            pos++;
781        }
782
783        void b64ConvertException(Exception e)
784            throws IOException // The Appendable throws this
785        {
786            if (log != null) log.append
787                ("\t\t" + C.RED + "Error Converting and Decoding Base-64 Image" + C.RESET + '\n');
788
789            skipped[pos]            = true;
790            exceptions[pos]         = e;
791            pos++;
792        }
793    }
794
795    /**
796        ******************************************************
797        class ImageScraper
798        ******************************************************
799        Iterable<String>                    source;
800        URL                                 originalPageURL;
801
802        String                              targetDirectory;
803        TargetDirectoryRetriever            targetDirectoryRetriever;
804        ImageReceiver                       imageReceiver;
805
806        ******************************************************
807        class ImageScraper.AdditionalParameters
808        ******************************************************
809        boolean                             skipOnIOException;
810        Predicate<URL>                      skipURL;
811        String                              fileNamePrefix;
812        boolean                             useDefaultCounterForImageFileNames;
813        FileNameRetriever                   getImageFileSaveName;
814        long                                maxDownloadWaitTime
815        TimeUnit                            waitTimeUnits
816    */
817
818    // *************************************************************************************
819    // *************************************************************************************
820    // download methods
821    // *************************************************************************************
822    // *************************************************************************************
823
824    /**
825     * Convenience Method.  Invokes {@link #download(AdditionalParameters, Appendable)}.
826     * <!-- NOTE: JavaDoc Upgrader REMOVES EXCEPTION THROWS... DO NOT MOVE THIS METHOD -->
827     */
828    public Results download()
829        throws IOException, MalformedURLException, URISyntaxException
830    { return download(null, null); }
831
832    /**
833     * This will iterate through the {@code URL's} and download them.  Note: Both the
834     * {@code AdditionalParameters} and {@code 'log'} parameters may be null, and if they are, they
835     * will be ignored.
836     *
837     * @param a This parameter takes customization requests for batch image downloads.  This 
838     * parameter can be passed 'null' and when it is, customizations shall be ignored.
839     * 
840     * <BR /><BR /><B>SKIP ON EXCEPTION:</B> The most useful feature of the {@code class 
841     * AdditionalParameters} is to facilitate a download where invalid or out-dated {@code URL's}
842     * do not cause the download mechanism to break - which normally would require running an
843     * image-download from the beginning.  There is a simple {@code AdditionalParameters}
844     * constructor that quickly builds an instance of that class to have {@code boolean
845     * skipOnIOException} initialized to <B>TRUE</B>.
846     * 
847     * @param log This shall receive text / log information.  If this parameter receives 'null',
848     * it will be ignored.
849     *
850     * <EMBED CLASS="external-html" DATA-FILE-ID="APPENDABLE">
851     *
852     * @return an instance of {@code class Results} for the download.  The {@code class
853     * ImageScraper.Results} contains several parallel arrays with information about images that
854     * have downloaded.  If an image-download happens to fail due to an improperly formed {@code
855     * URL} (or an 'incorrect' {@code URL}), then the information in the {@code Results} arrays 
856     * will contain a 'null' value for the index at those array-positions corresponding to the
857     * failed image.
858     *
859     * @throws IOException This might throw if there is an {@code IOException} when downloading an
860     * image, or attempting to save an image to the file-system.  If the
861     * {@code AdditionalParameters 'a'} parameter is set to suppress-exceptions (and continue to the
862     * next Image {@code URL}, via the {@code boolean skipIOExceptions}), then this exception will
863     * never throw.
864     *
865     * @throws MalformedURLException This will throw if there are problems de-referencing the
866     * {@code URL's}.  If the {@code AdditionalParameters 'a'} parameter is set to 
867     * suppress-exceptions (and continue to the next Image {@code URL}, via the {@code boolean
868     * skipIOExceptions}), then this exception will never throw.
869     *
870     * @throws URISyntaxException Same as {@code MalformedURLException.}  Will not throw if 
871     * exceptions are ignored.
872     */
873    public Results download(AdditionalParameters a, Appendable log)
874        throws IOException, MalformedURLException, URISyntaxException
875    {
876        // Compute the size of the input, will make array-building much faster
877        Counter counter = new Counter();
878        source.forEach(url -> counter.addOne());
879
880        Results     results = new Results(counter.size(), log);
881 
882        for (String src : source) 
883            if (src == null)
884            {
885                results.nullURL();
886                if ((a != null) && a.skipOnIOException) continue;
887                else throw new NullPointerException("One of the SRC URL's was null.");
888            }
889            else
890            {
891                Matcher m = IF.B64_INIT_STRING.matcher(src);
892                if (m.find())   CONVERT_B64(m.group(1), m.group(2), results, a);
893                else            DOWNLOAD(COMPUTE_URL(src, results, a), results, a);
894            }
895
896        return results;
897    }
898
899    // *************************************************************************************
900    // *************************************************************************************
901    // Internal COMPUTE-URL / FILENAME Methods
902    // *************************************************************************************
903    // *************************************************************************************
904
905    private void CONVERT_B64(
906            String imageFormatStr, String b64EncodedImage, Results results,
907            AdditionalParameters a
908        )
909        throws IIOException, IOException
910    {
911        if (results.log != null) 
912            results.log.append(
913                "\tBASE-64 IMAGE:\t" + imageFormatStr + ',' + b64EncodedImage.substring(0, 40) + '\n'
914            );
915
916        if ((a == null) || ((a != null) && a.skipBase64EncodedImages))
917        {
918            results.skipB64(imageFormatStr, b64EncodedImage.substring(0, 15));
919            return;
920        }
921
922        IF              ext;
923        BufferedImage   image;
924        try {
925            ext     = IF.get(imageFormatStr);
926            image   = IF.decodeBase64ToImage(b64EncodedImage, ext);
927            //image   = IF.decodeBase64ToImage_V2(b64EncodedImage, ext);
928        } catch (Exception e)
929        {
930            results.b64ConvertException(e);
931            if ((a != null) && a.skipOnIOException) return;
932            throw e;
933        }
934        String fileName = FILENAME(null, ext, results, a);
935        HANDLE_DOWNLOADED_IMAGE(null, fileName, ext, results, a, image);
936    }
937
938    private URL COMPUTE_URL(String src, Results results, AdditionalParameters a)
939        throws IOException, URISyntaxException
940    {
941        if (results.log != null)
942            results.log.append("\tChecking / Converting SRC-URL string:\t" + src + '\n');
943
944        if (StrCmpr.startsWithXOR_CI(src.trim(), "http://", "https://"))
945
946            try
947                { return new URL(URLs.toProperURLV7(src)); }
948
949            catch(MalformedURLException e)
950            {
951                results.urlException(src, e);
952                if ((a != null) && a.skipOnIOException) return null;
953                else throw e;
954            }
955            catch(URISyntaxException e)
956            {
957                results.urlException(src, e);
958                if ((a != null) && a.skipOnIOException) return null;
959                else throw e;
960            }
961
962        else if (originalPageURL == null)
963        {
964            MalformedURLException ex = new MalformedURLException(
965                "You have passed a null 'originalPageURL' parameter, but at least one of the URL's " +
966                "you have passed for downloading is either a partial URL, or else an invalid URL: " +
967                "[" + src + "]"
968            );
969
970            results.urlException(src, ex);
971
972            if ((a != null) && a.skipOnIOException) return null;
973            else                                    throw ex;
974        }
975
976        Ret2<URL, MalformedURLException> ret = Links.resolve_KE(src, originalPageURL);
977
978        if (ret == null) // I do not think this case is possible.  I'll leave it here anyway.
979        {
980            results.nullURL();
981            return null;
982        }
983        if (ret.b != null)
984        {
985            results.urlException(src, ret.b);
986            if ((a != null) && a.skipOnIOException) return null;
987            else throw ret.b;
988        }
989
990        // ADDED in NOVEMBER, 2019
991        // This micro-detail is the case where the "Resolved URL" also has ASCII-Escape characters
992        // that need to be escaped.  This is rare, but it needs to be heeded.  If there are
993        // ASCII-Escape character (which must be escaped).   Then "toProperURLV8" will handle that
994        // well enough.
995        //
996        // CONSIDER IT: Post-Processing of the "Resolve URLs" class
997
998        try
999            { return new URL(URLs.toProperURLV8(ret.a)); }
1000
1001        catch(MalformedURLException e)
1002        {
1003            results.urlException(src, e);
1004            if ((a != null) && a.skipOnIOException) return null;
1005            else throw e;
1006        }
1007        catch(URISyntaxException e)
1008        {
1009            results.urlException(src, e);
1010            if ((a != null) && a.skipOnIOException) return null;
1011            else throw e;
1012        }
1013    }
1014
1015    private String FILENAME(URL url, IF ext, Results results, AdditionalParameters a)
1016    {
1017        String fileName = ((a != null) && (a.fileNamePrefix != null)) ? a.fileNamePrefix : "";
1018
1019        if ((a != null) && (a.getImageFileSaveName != null))
1020            fileName = fileName + a.getImageFileSaveName.fileName
1021                (url, ext, results.pos, results.successCounter);
1022
1023        else if ((a == null) || ((a != null) && a.useDefaultCounterForImageFileNames))
1024            fileName = fileName + StringParse.zeroPad(results.successCounter);
1025
1026        else
1027        {
1028            fileName = url.getFile().substring(1);
1029            if (fileName.toLowerCase().endsWith('.' + ext.extension))
1030                fileName = fileName.substring(0, fileName.length() - 1 - ext.extension.length());
1031        }
1032        return fileName;
1033    }
1034
1035    // *************************************************************************************
1036    // *************************************************************************************
1037    // Internal Download Methods
1038    // *************************************************************************************
1039    // *************************************************************************************
1040
1041    /**
1042     * If this class has been used to make "multi-threaded" calls that use a Time-Out wait-period,
1043     * you might see your Java-Program hang for a few seconds when you would expect it to exit back
1044     * to your O.S. normally.
1045     *
1046     * <BR /><BR /><B><SPAN STYLE="color: red;">NOTE:</B></SPAN>
1047     * {@code AdditionalParameters.maxDownloadWaitTime, AdditionalParameters.waitTimeUnits} operate
1048     * by building a "Timeout &amp; Monitor" thread.  Thusly, when a program you have written
1049     * yourself reaches the end of its code, <I><B>if you have performed any time-dependent
1050     * Image-Downloads using {@code class ImageScraper}</B></I>, then your program <I>might not
1051     * exit immediately,</I> but rather sit at the command-prompt for anywhere between 10 and 30
1052     * seconds before this Timeout-Thread  dies.
1053     *
1054     * <BR /><BR /><B><SPAN STYLE="color: red">MULTI-THREADED:</B></SPAN> You may immediately
1055     * terminate any additional threads that were started using this method.
1056     */
1057    public static void shutdownTOThreads() { executor.shutdownNow(); }
1058
1059    // ******************************
1060    private static final    ExecutorService executor    = Executors.newCachedThreadPool();
1061    private static final    Lock            lock        = new ReentrantLock();
1062    // ******************************
1063
1064    private void DOWNLOAD(URL url, Results results, AdditionalParameters a) throws IOException
1065    {
1066        BufferedImage   image;
1067        if (url == null) return;
1068
1069        Appendable log = results.log;
1070        if (log != null) log.append("\tIMAGE-URL:\t\t" + url.toString() + '\n');
1071
1072        if ((a != null) && (a.skipURL != null) && (a.skipURL.test(url) == true))
1073        { results.skippedURL(url); return; }
1074
1075        // ******************************
1076        // *** ADDED on May 1st, 2019 ***
1077        // ******************************
1078        Callable<BufferedImage> threadDownloader = new Callable<BufferedImage>()
1079        {
1080            public BufferedImage call() throws Exception
1081            {
1082                try { return ImageIO.read(url); }
1083                catch (IIOException e)
1084                {   
1085                    // This will **sometimes** help when connecting to a URL "expects" this "User-Agent"
1086                    // This won't *always* work - or will it?  It is a very large-internet, with many MANY types of web-servers.
1087                    // THIS IS SORT-OF "ATTEMPT TO DOWNLOAD #2"
1088                    // try {
1089                    if (log != null) log.append("\tUSING USER-AGENT:\t" + url.toString() + '\n');
1090                    HttpURLConnection con = (HttpURLConnection) url.openConnection();
1091                    con.setRequestMethod("GET");
1092                    con.setRequestProperty("User-Agent", "Chrome/61.0.3163.100");
1093                    InputStream is = con.getInputStream();
1094                    return ImageIO.read(is);
1095                }
1096            }
1097        };
1098
1099        lock.lock();
1100        Future<BufferedImage> future = executor.submit(threadDownloader);
1101        lock.unlock();
1102
1103        long wt = ((a != null) && (a.waitTimeUnits != null)) 
1104            ? a.maxDownloadWaitTime 
1105            : MAX_WAIT_TIME;
1106
1107        TimeUnit tu = ((a != null) && (a.waitTimeUnits != null)) 
1108            ? a.waitTimeUnits
1109            : MAX_WAIT_TIME_UNIT;
1110
1111        try
1112            { image = future.get(wt, tu); }
1113
1114        catch (TimeoutException e)
1115        {
1116            if (e.getMessage() == null) e = new TimeoutException
1117                ("Waited: " + wt + " " + tu.toString());
1118
1119            results.downloadException(url, e);
1120
1121            if ((a != null) && a.skipOnIOException) return;
1122
1123            throw new IOException
1124                ("The download timed-out, see getCause() for more information.", e);
1125        }
1126
1127        catch (ExecutionException e)
1128        {
1129            Exception cause = ((e.getCause() != null) && (e.getCause() instanceof Exception)) 
1130                ? (Exception) e.getCause() 
1131                : e;
1132
1133            results.downloadException(url, cause);
1134
1135            if ((a != null) && a.skipOnIOException) return;
1136
1137            throw new IOException
1138                ("The download had an exception, see getCause() for more information.", cause);
1139        }
1140
1141        catch (InterruptedException e)
1142        {
1143            results.downloadException(url, e);
1144
1145            if ((a != null) && a.skipOnIOException) return;
1146
1147            throw new IOException(
1148                "The download was interrupted by another thread, see this Throwable.getCause() " +
1149                "for more information.", e
1150            );
1151        }
1152
1153        IF      ext         = IF.getGuess(url.toString());
1154        String  fileName    = FILENAME(url, ext, results, a);
1155
1156        HANDLE_DOWNLOADED_IMAGE(url, fileName, ext, results, a, image);
1157    }
1158
1159    private void HANDLE_DOWNLOADED_IMAGE(
1160            URL url, String fileName, IF ext, Results results,
1161            AdditionalParameters a, BufferedImage image
1162        )
1163        throws IIOException, IOException
1164    {
1165        Appendable log = results.log;
1166
1167        String dirName = null;
1168
1169        if (targetDirectory != null)
1170            dirName = targetDirectory;
1171
1172        else if (targetDirectoryRetriever != null)
1173            dirName = targetDirectoryRetriever.dir
1174                (url, fileName, ext, results.pos, results.successCounter);
1175
1176        else if (imageReceiver != null)
1177        {
1178            imageReceiver.save(url, fileName + '.' + ext.extension, ext, results.pos, results.successCounter, image);
1179            
1180            long l;
1181            {   // CODE COPIED FROM STACK-OVERFLOW.  This should probably become a separate-method. 
1182                //  I am not 100% it works yet.  (The "Green Check Mark" was not checked on this answer!)
1183                ByteArrayOutputStream tmp = new ByteArrayOutputStream();
1184                ImageIO.write(image, ext.extension, tmp);
1185                tmp.close();
1186                l = tmp.size();
1187            }
1188            results.imageReceiverSuccess(url, fileName, ext, l, image.getWidth(), image.getHeight());
1189            return;
1190        }
1191
1192        else throw new IllegalStateException
1193            ("Not image-target specified.  Illegal State - is a constructor overloaded?");
1194
1195        if (! dirName.endsWith(File.separator)) dirName += File.separator;
1196
1197        File f = null;
1198        if (ext != null)
1199            try {
1200                String fName = dirName + fileName + '.' + ext.extension;
1201                if (log != null) log.append("\tAttempting to save file: " + fName + '\n');
1202
1203                f = new File(fName);
1204                ImageIO.write(image, ext.extension, f);
1205                results.saveSuccess(
1206                    url, dirName, fileName, ext, f.length(),
1207                    image.getWidth(), image.getHeight()
1208                );
1209                return;
1210            } catch (Exception e)
1211            {
1212                results.saveFail(url, dirName, fileName, ext, e);
1213                if ((a != null) && a.skipOnIOException) return;
1214                throw e;
1215            }
1216        else
1217        {
1218            String fName = dirName + fileName + '.';
1219            for (IF imageFormat : IF.values())
1220                try {
1221                    f = new File(fName + imageFormat.extension);
1222                    ImageIO.write(image, imageFormat.extension, f);
1223                    results.saveSuccess(
1224                        url, dirName, fileName, imageFormat, f.length(),
1225                        image.getWidth(), image.getHeight()
1226                    );
1227                    return;
1228                }
1229                catch (javax.imageio.IIOException e)    { f.delete();   continue; }
1230                catch (Exception e)
1231                {
1232                    e.printStackTrace();
1233                    results.saveFail(url, dirName, fileName, imageFormat, e);
1234                    if ((a != null) && a.skipOnIOException) return;
1235                    throw e;
1236                }
1237        }
1238    }
1239
1240    // *************************************************************************************
1241    // *************************************************************************************
1242    // Localize Images methods
1243    // *************************************************************************************
1244    // *************************************************************************************
1245
1246    /** 
1247      * Convenience Method.  Invokes {@link #localizeImages(Vector, URL, Appendable, AdditionalParameters, String)}.
1248      * <BR /><BR />Passes null to {@link AdditionalParameters} and to root-{@code URL}.
1249      * <BR /><BR /><B>WARNING:</B> Presumes there are no partial-{@code URL's}
1250      */
1251    public static Ret2<int[], ImageScraper.Results> localizeImages
1252        (Vector<HTMLNode> page, Appendable log, String downloadDirectory)
1253        throws IOException
1254    { return localizeImages(page, null, log, null, downloadDirectory); }
1255
1256    /** 
1257      * Convenience Method.  Invokes {@link #localizeImages(Vector, URL, Appendable, AdditionalParameters, String)}.
1258      * <BR /><BR />Passes null to {@link AdditionalParameters}.
1259      */
1260    public static Ret2<int[], ImageScraper.Results> localizeImages
1261        (Vector<HTMLNode> page, URL pageURL, Appendable log, String downloadDirectory)
1262        throws IOException
1263    { return localizeImages(page, pageURL, log, null, downloadDirectory); }
1264
1265    /**
1266     * Downloads images located inside an HTML Page and updates the {@code SRC=...} {@code URL's}
1267     * so that the links point to a <I>local copy</I> of <I>local images</I>.
1268     *
1269     * <BR /><BR />After completion of this method, an HTML page which contained any HTML image
1270     * elements will have had those images downloaded to the local file-system, and also have had 
1271     * the HTML attribute {@code 'src=...'} changed to reflect the local image name instead of the
1272     * Internet URL name.
1273     *
1274     * @param page Any vectorized-html page or subpage.  This page should have HTML {@code <IMG ...>}
1275     * elements in it, or else this method will exit without doing anything.
1276     *
1277     * @param pageURL If any of the HTML image elements have {@code src='...'} attributes that are
1278     * partially resolved or <I>relative {@code URL's}</I> then this can be passed to the
1279     * {@code ImageScraper} constructors in order to convert partial or relative {@code URL's}
1280     * into complete {@code URL's.}  The Image Downloader simply cannot work with partially
1281     * resolved {@code URL's}, and will skip them if they are partially resolved.  This parameter
1282     * may be null, but if it is and there are incomplete-{@code URL's} those images will
1283     * simply not be downloaded.
1284     *
1285     * @param log This is the 'logger' for this method.  It may be null, and if it is - no output
1286     * will be sent to the terminal.
1287     *
1288     * <EMBED CLASS="external-html" DATA-FILE-ID="APPENDABLE">
1289     *
1290     * @param ap This is the {@link AdditionalParameters} parameter that allows to further
1291     * specify the request to the Image Downloader.  See the documentation for this class for more
1292     * information.  This parameter may be null, and if it is, it will be ignored and default
1293     * behavior will occur.
1294     * 
1295     * <BR /><BR /><B>SKIP ON EXCEPTION:</B> The most useful feature of the {@code class
1296     * AdditionalParameters} is to facilitate a download where invalid or out-dated {@code URL's}
1297     * do not cause the download mechanism to break - which normally would require running an
1298     * image-download from the beginning.  There is a simple {@code AdditionalParameters} 
1299     * constructor that quickly builds an instance of that class to have
1300     * {@code boolean skipOnIOException} initialized to <B>TRUE</B>.
1301     *
1302     * @param downloadDirectory This File-System directory where these files shall be stored.
1303     *
1304     * @return An instance of {@code Ret2<int[], ImageScraper.Results>}.  The two returned elements
1305     * of this class include:
1306     *
1307     * <BR /><BR /><UL CLASS="JDUL">
1308     * <LI> {@code Ret2.a (int[])}
1309     *      <BR /><BR />This shall contain an index-array for the indices of each HTML
1310     *      {@code '<IMG SRC=...>'} element found on the page.  It is not guaranteed that each of
1311     *      images will have been resolved or downloaded successfully, but rather just that an HTML
1312     *      {@code 'IMG'} element that had a {@code 'SRC'} attribute.  The second element of this
1313     *      return-type will contain information regarding which images downloaded successfully.
1314     *      <BR /><BR />
1315     * </LI>
1316     * <LI> {@code Ret2.b (ImageScraper.Results)}
1317     *      <BR /><BR />The second element of the return-type shall be the instance of
1318     *      {@link ImageScraper.Results} returned from the invocation of
1319     *      {@code ImageScraper.download(...)}.  This method will provide details about each of the
1320     *      images that were downloaded; or, if the download failed, the reasons for the failure.
1321     *      <I>This return element shall be null if no images were found on the page.</I>
1322     *      <BR />
1323     * </LI>
1324     * </UL>
1325     * 
1326     * <BR />These return {@code Object} references are not necessarily important - <I>and they
1327     * may be discarded if needed.</I>  They are provided as a matter of utility if further
1328     * verification or research into successful downloads is needed.
1329     *
1330     * @see AdditionalParameters
1331     */
1332    public static Ret2<int[], ImageScraper.Results> localizeImages(
1333        Vector<HTMLNode> page, URL pageURL, Appendable log, AdditionalParameters ap,
1334        String downloadDirectory
1335    )
1336        throws IOException
1337    {
1338        int[]               imgPosArr   = TagNodeFind.all(page, TC.Both, "img");
1339        Vector<TagNode>     vec         = new Vector<>();
1340
1341        // No Images Found.
1342        if (imgPosArr.length == 0) return new Ret2<int[], Results>(imgPosArr, null);
1343
1344        for (int pos : imgPosArr) vec.addElement((TagNode) page.elementAt(pos));
1345
1346        ImageScraper is = new ImageScraper(vec, pageURL, downloadDirectory);
1347        ImageScraper.Results r;
1348
1349        try
1350            { r = is.download(ap, log); }
1351        catch (URISyntaxException e)
1352        {
1353            throw new IOException(
1354                "There was a problem de-referencing one of the partial-URL's from the page URL.  " +
1355                "See this methods's Throwable.getCause() for details.",
1356                e
1357            ); 
1358        }
1359
1360        // ImageScraper.shutdownTOThreads(); 
1361        // NOTE-TO-READER: Need to call this method, or function will not shutdown.
1362        // NOTE: Commented out for now.
1363
1364        ReplaceNodes.r(page, imgPosArr, (HTMLNode n, int arrPos, int count) ->
1365        {
1366            if (    (r.fileNames[count] != null)
1367                &&  ((r.exceptions[count] == null)
1368                &&  (r.skipped[count] == false)))
1369
1370                return ((TagNode) page.elementAt(arrPos))
1371                        .setAV("src", r.fileNames[count], SD.SingleQuotes);
1372
1373            else
1374                return (TagNode) n;
1375        });
1376
1377        return new Ret2<int[], Results>(imgPosArr, r);
1378    }
1379}