|
| 1 | +--- |
| 2 | +layout: model |
| 3 | +title: InternVL 3 8B int4 |
| 4 | +author: John Snow Labs |
| 5 | +name: internvl3_8b_int4 |
| 6 | +date: 2025-05-17 |
| 7 | +tags: [en, open_source, openvino] |
| 8 | +task: Image Captioning |
| 9 | +language: en |
| 10 | +edition: Spark NLP 5.5.1 |
| 11 | +spark_version: 3.0 |
| 12 | +supported: true |
| 13 | +engine: openvino |
| 14 | +annotator: InternVLForMultiModal |
| 15 | +article_header: |
| 16 | + type: cover |
| 17 | +use_language_switcher: "Python-Scala-Java" |
| 18 | +--- |
| 19 | + |
| 20 | +## Description |
| 21 | + |
| 22 | +Visual Question Answering using InternVL. |
| 23 | + |
| 24 | +InternVLForMultiModal can load InternVL Vision models for visual question answering. |
| 25 | +The model consists of a vision encoder, a text encoder, a text decoder and a model merger. |
| 26 | +The vision encoder will encode the input image, the text encoder will encode the input text, |
| 27 | +the model merger will merge the image and text embeddings, and the text decoder will output the answer. |
| 28 | + |
| 29 | +InternVL 2.5 is an advanced multimodal large language model (MLLM) series that builds upon InternVL 2.0, |
| 30 | +maintaining its core model architecture while introducing significant enhancements in training and testing |
| 31 | +strategies as well as data quality. Key features include: |
| 32 | +- Large context window support |
| 33 | +- Multilingual support |
| 34 | +- Multimodal capabilities handling both text and image inputs |
| 35 | +- Optimized for deployment with int4 quantization |
| 36 | + |
| 37 | +{:.btn-box} |
| 38 | +<button class="button button-orange" disabled>Live Demo</button> |
| 39 | +<button class="button button-orange" disabled>Open in Colab</button> |
| 40 | +[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/internvl3_8b_int4_en_5.5.1_3.0_1747457622170.zip){:.button.button-orange.button-orange-trans.arr.button-icon} |
| 41 | +[Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/internvl3_8b_int4_en_5.5.1_3.0_1747457622170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} |
| 42 | + |
| 43 | +## How to use |
| 44 | + |
| 45 | + |
| 46 | + |
| 47 | +<div class="tabs-box" markdown="1"> |
| 48 | +{% include programmingLanguageSelectScalaPythonNLU.html %} |
| 49 | +```python |
| 50 | + |
| 51 | +python |
| 52 | +import sparknlp |
| 53 | +from sparknlp.base import * |
| 54 | +from sparknlp.annotator import * |
| 55 | +from pyspark.ml import Pipeline |
| 56 | +from pyspark.sql.functions import lit |
| 57 | + |
| 58 | +image_df = spark.read.format("image").load(path=images_path) # Replace with your image path |
| 59 | +test_df = image_df.withColumn("text", lit("<|im_start|><image> |
| 60 | +Describe this image in detail.<|im_end|><|im_start|>assistant |
| 61 | +")) |
| 62 | + |
| 63 | +imageAssembler = ImageAssembler() |
| 64 | + .setInputCol("image") |
| 65 | + .setOutputCol("image_assembler") |
| 66 | + |
| 67 | +visualQAClassifier = InternVLForMultiModal.pretrained() |
| 68 | + .setInputCols("image_assembler") |
| 69 | + .setOutputCol("answer") |
| 70 | + |
| 71 | +pipeline = Pipeline().setStages([ |
| 72 | + imageAssembler, |
| 73 | + visualQAClassifier |
| 74 | +]) |
| 75 | + |
| 76 | +result = pipeline.fit(test_df).transform(test_df) |
| 77 | +result.select("image_assembler.origin", "answer.result").show(False) |
| 78 | + |
| 79 | +``` |
| 80 | +```scala |
| 81 | + |
| 82 | +import spark.implicits._ |
| 83 | +import com.johnsnowlabs.nlp.base._ |
| 84 | +import com.johnsnowlabs.nlp.annotator._ |
| 85 | +import org.apache.spark.ml.Pipeline |
| 86 | +import org.apache.spark.sql.DataFrame |
| 87 | +import org.apache.spark.sql.functions.lit |
| 88 | + |
| 89 | +val imageFolder = "path/to/your/images" // Replace with your image path |
| 90 | + |
| 91 | +val imageDF: DataFrame = spark.read |
| 92 | + .format("image") |
| 93 | + .option("dropInvalid", value = true) |
| 94 | + .load(imageFolder) |
| 95 | + |
| 96 | +val testDF: DataFrame = imageDF.withColumn("text", lit("<|im_start|><image> |
| 97 | +Describe this image in detail.<|im_end|><|im_start|>assistant |
| 98 | +")) |
| 99 | + |
| 100 | +val imageAssembler: ImageAssembler = new ImageAssembler() |
| 101 | + .setInputCol("image") |
| 102 | + .setOutputCol("image_assembler") |
| 103 | + |
| 104 | +val visualQAClassifier = InternVLForMultiModal.pretrained() |
| 105 | + .setInputCols("image_assembler") |
| 106 | + .setOutputCol("answer") |
| 107 | + |
| 108 | +val pipeline = new Pipeline().setStages(Array( |
| 109 | + imageAssembler, |
| 110 | + visualQAClassifier |
| 111 | +)) |
| 112 | + |
| 113 | +val result = pipeline.fit(testDF).transform(testDF) |
| 114 | + |
| 115 | +result.select("image_assembler.origin", "answer.result").show(false) |
| 116 | + |
| 117 | +``` |
| 118 | +</div> |
| 119 | + |
| 120 | +{:.model-param} |
| 121 | +## Model Information |
| 122 | + |
| 123 | +{:.table-model} |
| 124 | +|---|---| |
| 125 | +|Model Name:|internvl3_8b_int4| |
| 126 | +|Compatibility:|Spark NLP 5.5.1+| |
| 127 | +|License:|Open Source| |
| 128 | +|Edition:|Official| |
| 129 | +|Input Labels:|[image_assembler]| |
| 130 | +|Output Labels:|[answer]| |
| 131 | +|Language:|en| |
| 132 | +|Size:|7.9 GB| |
0 commit comments