\documentclass[lettersize,journal]{IEEEtran}
\usepackage{amsmath,amsfonts}
\usepackage{algorithmic}
\usepackage{algorithm}
\usepackage{array}
\usepackage{booktabs}
\usepackage{adjustbox}
\usepackage{multirow}
\usepackage[caption=false,font=normalsize,labelfont=sf,textfont=sf]{subfig}
\usepackage{textcomp}
\usepackage{stfloats}
\usepackage{url}
\usepackage{verbatim}
\usepackage{graphicx}
\usepackage{threeparttable}
\usepackage{cite}
\hyphenation{op-tical net-works semi-conduc-tor IEEE-Xplore}
% updated with editorial comments 8/9/2021
\begin{document}
\title{Robust Text Image Manipulation Localization Against Distortion}
\author{Haiming Yao~\IEEEmembership{Student Member,~IEEE,} Wenyong Yu\IEEEmembership{Member,IEEE,} Wei Luo~\IEEEmembership{Student Member,~IEEE,} \
Zhenfeng Qiang, Donghao Luo, and Xiaotian Zhang
% <-this % stops a space
}
% Remember, if you use this you must call \IEEEpubidadjcol in the second
% column for its text to clear the IEEEpubid mark.
\maketitle
\begin{abstract}
This paper presents a novel framework, named Global-Local Correspondence Framework (GLCF), for visual anomaly detection with logical constraints. Visual anomaly detection has become an active research area in various real-world applications, such as industrial anomaly detection and medical disease diagnosis. However, most existing methods focus on identifying local structural degeneration anomalies and often fail to detect high-level functional anomalies that involve logical constraints. To address this issue, we propose a two-branch approach that consists of a local branch for detecting structural anomalies and a global branch for detecting logical anomalies. To facilitate local-global feature correspondence, we introduce a novel semantic bottleneck enabled by the visual Transformer. Moreover, we develop feature estimation networks for each branch separately to detect anomalies. Our proposed framework is validated using various benchmarks, including industrial datasets, Mvtec AD, Mvtec Loco AD, and the Retinal-OCT medical dataset. Experimental results show that our method outperforms existing methods, particularly in detecting logical anomalies.
The TITD task involves detecting tampering in text and identifying instances where text has been erased from empty regions. This is a tough process since erased text may leave minor traces or abnormalities that are difficult to identify using conventional approaches.
\end{abstract}
\begin{IEEEkeywords}
Anomaly detection, anomaly localization, Global-Local correspondence, Semantic bottleneck, Vision Transformer
\end{IEEEkeywords}
\section{Introduction}
\IEEEPARstart{T}{ext} images are increasingly used to communicate information, especially in digital finance, electronic commerce, security audit, and qualification review. The legitimacy of them is seldom discussed, which has recently raised worries regarding information security. Due to the importance of the information contained in texts, it is essential to prevent their alteration. Even modest alterations to a sentence can have a substantial impact on its overall meaning. In recent years, both academia and industry have sought to prevent the malicious manipulation of images by focusing on image forensics. Text is frequently presented in an unstructured manner, and tampered regions can be very small or have low contrast compared to their environs, which makes detecting tampered text more difficult than detecting tampered images.
\begin{figure}[t]\centering
\includegraphics[width=8.8cm]{imgs/before_after_compression2.pdf}
\caption{(a) A toy example illustrating two types of anomalies: local structural anomalies and global logical anomalies. (b) Comparison of detection performance between the existing state-of-the-art method Draem \cite{r20} and our proposed GLCF method for local structural anomalies and global logical anomalies.}
\label{FIG1}
\end{figure}
However, previous research in document analysis and recognition has been primarily directed towards detecting and comprehending the content of textual information, disregarding the verification of the authenticity of text images. Meanwhile, the majority of image forensics research has focused on identifying and localizing manipulated regions in natural images, while analyzing non-natural images has received little attention. Particularly, document text images have been largely neglected in current research and thus require a more concentrated and thorough examination.
Based on the comparison between (a) and (b) in Figure 1, the state-of-the-art model for detecting tampering in natural images does not demonstrate optimal performance when applied to textual images.
As image forensics continues to transition from basic research to practical applications, it is observed that most existing image forensic algorithms have been developed and validated exclusively for ideal controlled laboratory settings, without due consideration of their robustness and generality in real-world environments.
It is commonly believed that the altered image to be inspected has not been subjected to any pre- or post-processing. However, in real-world circumstances, forgeries photographs are frequently post-processed to conceal signs of tampering. In a real-world scenario, for example, the doctored image may be further processed or sent across a social network channel with unknown distortion, significantly degrading the forgery detection performance. Compared with (b) and (c) in Figure 1, after image blending, the robustness of the state-of-the-art method for detecting text image tampering needs improvement, as demonstrated by the decrease in its performance.
For detecting and localizing tampering in text images, it is essential to develop a robust model that can locate minor tampered areas and is resistant to various unknown distortion. In this paper, we propose a robust text image manipulation localization against distortion scheme.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%need to polish
Our motivation is based on the following assumptions: In terms of text location and geometric structure, the altered and original texts occupy the same semantic space. However, local texture changes can be seen between tampered and authentic texts. Tampered letters have smoother textures than their genuine equivalents. The detection of image manipulation is based on the discovery of different local structural relationships between pixels. Any manipulation operation performed on an image will change such local relationships.
Hence, it is crucial for Text Image Tampering Detection (TITD) methods to enhance the discriminative ability of class-specific texture features while preserving semantic invariance. This optimization enables effective differentiation between tampered and authentic texts.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Specifically, we suggest a novel network architecture for image enhancement that can recover the distorted tampering traces. The main idea behind our work is to adaptively enhance the image features dynamically conditioned on themselves. This allows the enhancement of the network to selectively enhance only those features that are useful for text image tamper detection. Besides, We propose an encoder-decoder manipulation localization network architecture consisting of two modules. The first module captures low-level and high-level feature maps from multi-scale and fuses them, while the second module aims to refine the forgery detection results using the proposed attention mechanism. The created datasets of text images, as well as the SACP dataset~\cite{Zhuang2022ReLocAR}, are utilized to conduct comparative experiments and ablation studies.
Experimental results demonstrate that the proposed method outperforms state-of-the-art techniques, demonstrating its efficacy and robustness in detecting text image manipulation. The contributions of this work can be summarized as follows:
\begin{enumerate}
\item We propose the presented architecture for deep forgery detection incorporates a multi-scale feature extraction module and two attention modules that function to extract low-level and global-level features. Furthermore, a cross-feature module is proposed to integrate the features arising from different levels, capture shared features, suppress background noise in complex regions, and compensate for the loss of features at varying levels.
\item Inspired by ReLoc~\cite{Zhuang2022ReLocAR} , we propose a pre-processing module that serves to augment distorted tampering traces in document images. With the aid of our proposed module, subsequent tamper detection modules are empowered to heighten the discriminability of tampered and legitimate regions. Overcoming the challenges posed by intricate background textures and post-processing manipulations, our method mitigates their impact on forgery detection to a reduced or eliminated extent. Due to the uncertainty of the distortion type in most real scenes, a need for a flexible blind approach toward enhancing tampering traces arises.
\item We provide a large-scale dataset featuring a variety of tampering methods and scenarios which can bolster tampered text detection research efforts. Through the inclusion of several blending approaches, a forgery document images dataset is established to elevate the level of difficulty and authenticity within this dataset. Furthermore, an innovative data synthesis approach is introduced that can generate realistic tampered documents using unlabeled document images with greater efficiency.
\end{enumerate}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%need to polish
The rest of the paper is organized as follows. Section II presents a brief review of text image tampering and the method of text image forensics. The proposed framework, including an encoder, decoder, and attention mechanism, is described in Section III. Section IV introduces our forgery document images dataset and shows our experimental results, and we conclude this paper in Section V.
\section{Related work}
Text image manipulation is a constantly growing field, and as attackers come up with more advanced ways to change text in images, new difficulties and solutions pop up.
\subsection{Text Editing and Tampering}
There have emerged some deep learning-based text editing algorithms which alter the textual information of an image in an end-to-end fashion.
ETW~\cite{Wu2019EditingTI} proposes an end-to-end network for text editing tasks that can substitute text from a scene text image while preserving the original style. To accomplish this task, they divided it into three steps: (1) extract the foreground text style and apply it to the input text using a skeleton; (2) erase the style image with the suitable texture to get the background image; and (3) integrate the transferred text with the erased backdrop. In order to improve the performance of ETW~\cite{Wu2019EditingTI}, SwapText ~\cite{Yang2020SwapTextIB} presents a text-image swapping scheme (SwapText) for scenes, focusing particularly on the quality of the text in images with perspective and curvature. STEFANN~\cite{Roy2019STEFANNST} proposes a character-level text editing network to modify individual characters in text images. Chen's attack approach has successfully breached a number of complex document authentication systems, and the development of text editing technology based on deep learning poses significant security issues for the use of document images.
In recent years, there has been a rise in the amount of research interest within the computer vision field on text removal. Its purpose is to do away with the text and replace it with stuff that makes more sense. Text removal can aid in the prevention of privacy leaks by allowing users to conceal some private communications. On the other hand, it can also be used for nefarious purposes, like as wiping important information or tampering with information.
\subsection{ Text Image Forensics}
Despite the fact that text manipulation technologies have advanced in recent years, the topic of tampered text detection and localization approaches is almost unexplored. The majority of prior research in Document Analysis and Recognition has focused on identifying and comprehending text content. Nonetheless, its veracity is rarely discussed, prompting significant worries about the security of information in daily life. In recent years, both academics and business have increased their focus on image forensics in an effort to guard against malicious image manipulations. Most of the studies look at natural images, and the things that have been changed are usually things like people or cars. While it's more challenging to locate changed texts because they have unique characteristics. The document images contain distinctive features, such as uniform textures, modest closed borders, and regular gaps between objects, which set them apart from natural image fraud detection. Text images are hard to identify the authenticity using natural image forgery techniques.
Cruz et al.~\cite{LocalBP} employed consistent Local Binary Patterns (LBP) for textural characteristics that are typical of altered regions. They employed a Support Vector Machine (SVM) to determine if document image patches were altered. A unique two-stream network design was proposed by Xu et al. ~\cite{Xu2021DocumentIF}. SIEN is one stream that learns spatial information directly from document image patches. CFEN with skip connections is another stream that learn abnormal traits. Sun et al. propose an efficient encoder-decoder network, MFAN~\cite{Sun2022MFANMF}, for the identification of faked certificate images. By fusing the final feature map based on low-level convolution layers, more information is preserved and problems in locating little tampering objects are avoided. Keyu Sunet et al. ~\cite{Sun2019DifferentialAT} present an effective and efficient approach for passively detecting digital document image manipulation. The anomaly in the spatial-domain difference image is found and utilized to determine splicing boundaries. Wang et al. ~\cite{Wang2022DetectingTS} propose a new task, Tampered Scene Text Detection, to detect text occurrences and texture authenticity. They address the utility of frequency information in feature learning and suggest a parallel-branch feature extractor to improve feature representation capabilities. Nevertheless, the work solely focuses on word-level manipulated text detection. The altered cases at the character and line levels are not included.
\subsection{Robustness against distortion}
ReLoc~\cite{Zhuang2022ReLocAR} cascades a restoration module after a localization module in order to increase resilience. The goal of the restoration module is to take a distorted image and restore it to its original, high-quality state by fixing the tampering traces. They introduce three different losses to optimize the restoration module. Unfortunately, the network
in restoration, module is not especially intended to boost tamper detection performance.
%%%%%%%%%%%% Pic_9
\begin{figure}[htb!]
\centering
\includegraphics[width=0.48\textwidth,height=0.25\textwidth]{imgs/s38.pdf}
\caption{The workflow for constructing forgery image samples with image blending, along with its ground-truth label.}
\label{dataset_pic}
\end{figure}
\begin{figure}[htb]
\centering
\includegraphics[scale=0.29]{imgs/sketch.pdf}
\caption{Fusing features of different levels. (c) represents the low level features. (d) means the high level features. (e) is the fused features by F\(^3\)Net. Clearly, the fused features have clear boundaries as well as few background noises.}
\label{fusion_feature}
\end{figure}
\section{Approach}
\begin{figure}[htb]
\centering
\includegraphics[scale=0.52]{imgs/framework.pdf}
\caption{An overview of proposed F\(^3\)Net. ResNet-50 is used as the backbone encoder. Cross feature module (CFM) is used as the basic module to fuse features of different layers. Cascaded feedback decoder (CFD) contains multiple sub-decoders to feedback and refine multi-level features. Multi-level supervision (MLS) helps to ease the optimization of F\(^3\)Net.}
\label{framework}
\end{figure}
\subsection{Method Overview}
\subsection{Enchnen module}
\begin{figure}[t]\centering
\includegraphics[width=8.8cm]{imgs/s16.pdf}
\caption{Comparison with the traditional ViT for classification and the proposed SAM. (a) Traditional ViT\cite{r29} introduces a single class token as an aggregation of global semantics for image classification. (b) Our SAM introduces an additional set of semantic tokens to generate global semantic representation with spatial information, while preserving the original tokens containing original information.}
\label{FIG0}
\end{figure}
\subsection{Proposed Encoder}
A fundamental concept in image manipulation detection revolves around the presence of specific local structural relationships among pixels that are independent of the image content~\cite{Xu2021DocumentIF}. Image manipulation operations disrupt these local relationships. Therefore, in order to detect operations similar to those performed on certificate documents, the feature extractor for image operation detection must learn the relationships between pixels within their local domains while suppressing the influence of image content to avoid capturing content-related features. Employing a deeper architecture would result in the loss of tampering information in small areas due to the comparatively simple background and texture aspects of document images. The existing general approaches for detecting image manipulation, which are based on convolutional neural networks (CNNs), are inadequate for detecting tampering in certificate files. This inadequacy arises due to the CNN-based methods predominantly extracting image content features, while the tampered regions in most certificate file types exhibit weak correlation with image content. Consequently, when directly applied to certificate document processing, the effectiveness of current image processing detection algorithms diminishes.
Hence, we select EfficientNet-B3 as the encoder to extract additional feature data during the encoding phase. EfficientNet-B3 represents a simple and efficient neural network composite scaling method ~\cite{Tan2019EfficientNetRM} that considers all three dimensions of depth, width, and resolution. The architecture of EfficientNet encompasses MB Conv (mobile inverted bottleneck convolution)~\cite{Howard2019SearchingFM}, which incorporates squeeze and excitation optimizations as the foundational building block, whose structure is shown in Figure \ref{FIG3}. The MBConv structure comprises four convolutional layers: the expansion convolutional layer, the depthwise convolutional layer, the sequence-and-exception layer, and the point convolutional layer. The feature map is elevated from low to high dimensions using an expansion convolutional layer, which aids in preventing overfitting and improving generalization. The deep and point convolutional layers facilitate the extraction of more spatial and channel feature information. By leveraging the squeeze and excitation layer, different channels are assigned weights to suppress those containing less critical information, thereby emphasizing pixels corresponding to target categories. Finally, a skip-connection operation is employed between the output and input to preserve information concerning small forgery areas even after pooling, thus preventing its loss. The encoder framework of our proposed model is illustrated in Figure \ref{FIG4}. This framework enables the effective detection of image forgery regions, as the forged parts are emphasized in the generated feature maps.
\begin{figure}[t]
\centering
\includegraphics[width=8.8cm]{imgs/s3.pdf}
\caption{Comparison with the traditional ViT for classification and the proposed SAM. (a) Traditional ViT\cite{r29} introduces a single class token as an aggregation of global semantics for image classification. (b) Our SAM introduces an additional set of semantic tokens to generate global semantic representation with spatial information, while preserving the original tokens containing original information.}
\label{FIG3}
\end{figure}
\begin{figure}[t]
\centering
\includegraphics[width=8.8cm]{imgs/s4.pdf}
\caption{Comparison with the traditional ViT for classification and the proposed SAM. (a) Traditional ViT\cite{r29} introduces a single class token as an aggregation of global semantics for image classification. (b) Our SAM introduces an additional set of semantic tokens to generate global semantic representation with spatial information, while preserving the original tokens containing original information.}
\label{FIG4}
\end{figure}
\subsection{Proposed Decoder}
\begin{figure}[htb]
\centering
\includegraphics[scale=0.52]{imgs/s25.pdf}
\caption{An overview of proposed F\(^3\)Net. ResNet-50 is used as the backbone encoder. Cross feature module (CFM) is used as the basic module to fuse features of different layers. Cascaded feedback decoder (CFD) contains multiple sub-decoders to feedback and refine multi-level features. Multi-level supervision (MLS) helps to ease the optimization of F\(^3\)Net.}
\label{framework}
\end{figure}
In this work, we propose an attention module Low-level and High-level Squeeze-and-Excitation (LHSE), which aims to selectively integrate features. By combining the features from low-level and high-level channel maps, we address the issue of information loss caused by convolutional blocks and upsampling processes. This integration compensates for the discarded information and enhances the representation capabilities of the model.
Furthermore, we propose a cascaded decoder that refines multi-level features to improve the accuracy of detecting and locating small forgery areas. The cascaded decoder operates iteratively to refine the feature representations, gradually improving the detection and localization maps. This approach enables the model to capture finer details and achieve more precise results.
\begin{figure}[!t]
\centering
\includegraphics[width=0.5\textwidth,height=0.2\textwidth]{imgs/s32.pdf}
\caption{The architecture diagram for our proposed LHSE attention module}
\label{pic_lhse_attention _block}
\end{figure}
\textbf{LHSE Module}: As shown in Fig. \ref{pic_lhse_attention _block}, specifically, The FIA module takes three inputs: high-level input features \(F_{O}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\) from the GEAM attention module and low-level features from the corresponding bottom layer.
The fusion of high-level and low-level features is performed through a squeeze and excitation operation. The high-level features are up-sampled and processed to obtain a response map \(Z_{1}\) \(\in\) \(\mathbb{R}^{1 \times 1 \times 256}\), while the low-level features also generate a response map \(Z_{2}\) \(\in\) \(\mathbb{R}^{1 \times 1 \times 256}\) through squeeze and excitation. These two response maps are then combined using an element-wise addition operation.
\begin{equation}
Z_\text{total} = Z_{1}+ Z_{2},
\label{z_eq6}
\end{equation}
The resulting combined feature map \(Z_\text{total}\) \(\in\) \(\mathbb{R}^{1 \times 1 \times 256}\), is further used in a channel-wise multiplication operation with the high-level input features \(F_{O}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\) .
\begin{equation}
F_\text{LHSE} = F_{O}\otimes Z_\text{total},
\label{z_eq28}
\end{equation}
where \(\otimes\) is a channel-wise multiplication. This multiplication operation enhances the valuable features and suppresses less useful features, effectively integrating high-level features with spatial information.
low-level features usually have larger spatial size and more fine-grained details, while high-level features tend to gain more semantic knowledge and discard some meaningless or irrelevant detail information. Generally speaking, the high-level features can provide abstract semantic information, which are beneficial to the coarse localization of salient objects, whereas the low-level features that contain the spatial structural details are suitable to refine boundaries.
By leveraging both high-level semantic information and low-level spatial details, the LHSE module aims to improve the network's ability to accurately localize objects with fine-grained details. These are exploited to guide the network to concentrate on the valuable features and suppress the less useful feature for splicing localization. In this way, properties of a tiny target will not lost after pooling, and localization performance for a small region is improved.
%%%%%%%%%%%%%%
In the FIA module, the low-level detail information and high-level semantic information are fused in an interweaved way.
Attention to forging attempts to capture noise feature interdependence in spatial and channel dimensions. We extract spatial relationships of noise features in every pixel pair of feature maps in the spatial dimension. Similarly, we capture noise feature channel relationships between channel pairs in the channel dimension.
%%%%%%%%%%%%%%
\textbf{GEAM Attention Module}:
%%%%%%%%%%%%%%
For the challenging scenarios in salient object detection, such as cluttered background, foreground disturbance, and multiple salient objects, simple integration of high-level and low-level features may fail to completely detect the salient regions due to lacking the global semantic relationship among different parts of salient object or multiple salient objects.
%%%%%%%%%%%%%%
After the multi-scale information module, we use a Global view and Edge Attention Module (GEAM) to learn how pixels are related to each other over a long-range, increasing receptive fields and rendering targets more clearly. By learning the adaptive re-calibration of the extracted features, it pays closer attention to the position of the valuable pixels and learns more competitive features on manipulated regions and the original regions. Intuitively, the edge shape is more robust than the texture in document image forensics. It is possible to detect some tampering since boundaries are hidden by image blending. So the edge features are fused to enhance boundary-related information in our attention module.
As shown in Fig. \ref{pic_GEAM_attention _block}, specifically, given a local feature map \(F_{aspp}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\) generated by ASPP module as input, then we feed it into two \(1 \times 1\) convolution layer to obtain \(F_{A}\) \(\in\) \(\mathbb{R}^{H \times W \times 64}\) and \(F_{B}\) \(\in\) \(\mathbb{R}^{H \times W \times 64}\). Then \(F_{A}\) and \(F_{B}\) are reshaped into \(F_{A}{'}\) \(\in\) \(\mathbb{R}^{N \times 64}\) and \(F_{B}{'}\) \(\in\) \(\mathbb{R}^{N \times 64}\) , where $N = H \times W $ which is the number of pixels. A spatial attention feature map \(S \in \mathbb{R}^{N \times N}\) is then obtained using the softmax function.
\begin{equation}
S_{j i}=\frac{\exp \left(F_{A_i}F_{B_j}\right)}{\sum_{i=1}^{N} \exp \left(F_{A_i}F_{B_j}\right)}.
\label{s_eq3}
\end{equation}
The \(S_{j i}\) denotes the impact of the \(i\)-th position on the \(j\)-th position in the feature map. With multiplication, the network learns the long-range dependencies within the feature map in the feature extraction process and increases the receptive field.
Meanwhile, we reshape \(F_{aspp}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\) to \(F^{'}_{encoder}\) \(\in\) \(\mathbb{R}^{N \times 256}\) . Then a matrix multiplication between the spatial attention map \(S_{softmax}\) \(\in\) \(\mathbb{R}^{N \times N}\) and \(F^{'}_{encoder}\) \(\in\) \(\mathbb{R}^{N \times 256}\) is performed to obtain the re-weight feature map \(P_{encoder}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\):
\begin{equation}
P_\text{encoder} =S_\text{softmax} \cdot F^{'}_\text{encoder}.
\label{p_eq4}
\end{equation}
In addition, the a edge feature map \(F_{E}\) \(\in\) \(\mathbb{R}^{H \times W \times 1}\) which is generated by canny algorithm is concatenated with to enhance forgery feature. Finally, we obtain the final output \(F_{O}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\) by a \(3 \times 3\) convolution layer. Specifically, the final output \(F_{O}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\) is calculated as following formula \ref{f_eq5}:
\begin{equation}
F_{O}= \omega\left(\text{concat}\left[P_\text{encoder},F_{E}\right]\right),
\label{f_eq5}
\end{equation}
where \(\omega\left(\right)\) is a \(3 \times 3\) convolution layer. Thus, the final output \(F_{O}\) \(\in\) \(\mathbb{R}^{H \times W \times 256}\) of GEAM attention module combines global contextual information with local feature maps according to the spatial attention map.
In short, the proposed attention module can automatically focus on the target area, suppress the response of irrelevant regions, and highlight the feature information crucial to a forgery region.
\subsection{Loss Function}
The primary objective of image forgery detection is the precise localization of manipulated regions, achievable through pixel-wise classification. Tampered text in document images often exhibits limited visual tampering cues and occupies relatively small areas. Document images typically share a uniform background color, while text within clusters typically adheres to the same font and size. Moreover, the use of multiple convolution blocks and upsampling operations may result in the loss of significant information, diminishing the effectiveness of supervision. To enhance detection performance, we propose the adoption of a Weight-Dice-BCE loss function, which combines the Weight Cross-Entropy (WeightBCE) and Weight Dice (WeightDice) loss functions. The total loss is defined as:
\begin{equation}
L_\text{total} = \lambda_{1} L_\text{WeightBCE}+ \lambda_{2}L_\text{WeightDice}. \label{eq10}
\end{equation}
The Weight-BCE loss function is a modified version of the Binary Cross-Entropy (BCE) loss that addresses the imbalance issue between background and foreground pixels by assigning different weights to positive and negative samples. To adjust the importance of positive labels, we introduce an additional weight parameter, q, which is set to 0.2, indicating a negative-to-positive sample ratio of 0.2. The selection of these parameters is based on ablation experiments detailed in the subsequent section. The Weight-BCE loss is computed as follows:
\begin{equation}
L_\text{Weight-BCE} = -q \times y \log \left(\hat{y}\right) - \left(1-y\right) \log \left(1-\hat{y}\right),
\end{equation}
where \(y\) is the ground truth label and \(\hat{y}\) is the predicted probability.
The Weight-Dice loss calculates the Dice coefficient by considering the intersection and union between the predicted and ground truth segmentation masks. By incorporating weights, it places emphasis on the accurate detection and classification of forgery regions, including smaller ones. The Weight-Dice loss is given by:
\begin{equation}
L_\text{Weight-DICE} = 1 - \frac{{2\times \sum(p \times \text{intersection}_i)}}{{\sum(p \times \text{union}_i)}},
\end{equation}
In both loss functions, the assignment of weights is crucial for effectively balancing the impact of different classes during the optimization process. By assigning higher weights to underrepresented classes, the loss functions ensure that the model focuses on detecting and accurately classifying the minority class (forged images) while also considering the majority class (authentic images).
\section{Experiments}
\subsection{Implementation Details And Evaluation Metrics}
\textbf{Implementation Details:} In this work, a wide receptive field is employed to effectively capture the context information from the input image due to the minimal visual difference between tampered and non-tampered regions. Specifically, the original image is randomly augmented twice, resulting in an enhanced representation that emphasizes local details and boosts the model's ability to discern abnormal regions within tampered areas, such as the removal of existing residues or the introduction of blurred edges during pasting operations.
The proposed model is trained using a NVIDIA Tesla V100 GPU and implemented within the PyTorch framework. Training is performed on 512x512 images with a batch size of 16 over 100 iterations, incorporating data augmentation techniques. Optimization is achieved using the AdamW optimizer [15] with a fixed learning rate of 0.0001, a momentum of 0.9, and a weight decay of 0.0005.
To initialize and fine-tune the encoder, pre-trained weights from EfficientNet-B3 on ImageNet are utilized. The loss weights for the combined loss function are adjusted to 1 = 0.7 and 1 = 0.3. Furthermore, the Test Time Augmentation (TTA) approach is employed during the prediction phase to enhance the localization accuracy of tampered regions. This technique involves generating predictions for a test image and its various transformations, followed by merging the results. The employed transformations include horizontal flipping, vertical flipping, 90-degree rotation, 180-degree rotation, and 270-degree rotation.
\textbf{Evaluation Metrics:}
Fundamentally, image splicing localization may be viewed as a problem with binary classification at the pixel level, necessitating the evaluation of our model using benchmark metrics such as the F1 score and intersection over union (IoU). The IoU metric expresses the level of overlap between the produced candidate boundary and the ground truth boundary as the ratio of their intersection to their union. A greater IoU number suggests superior performance. The F1 score and IoU may be determined using the following equations:
\begin{equation}
F_{1}=\frac{2TP}{2 TP+F N+FP},
\label{eq12}
\end{equation}
\begin{equation}
\mathrm{IoU}=\frac{TP}{TP+FN+FP},
\label{eq13}
\end{equation}
where TP and TN represent true positive and true negative, respectively, and indicate the pixels that have been accurately recognised as being tampered with or legitimate. In a similar vein, the abbreviations FP and FN stand for false positive and false negative, respectively, and refer to misidentified real and manipulated pixels.
Beside, in order to create a trustworthy text image forgery detection system, it is imperative to minimize the False Negative Rate (FNR). In other words, the number of images that are incorrectly identified as authentic when they are actually forged should be as low as possible. This necessity is especially important in forensic and security settings, where overlooking a tampered image can have severe consequences. It is calculated as given below:
\begin{equation}
FNR =\frac{FN}{FN+TP}.
\label{eq12}
\end{equation}
\subsection{Dataset Generation}
The field of image tampering detection is making strides in identifying manipulations in natural images, but this progress falls short of addressing the true scope of the problem. In reality, many fraudulent images, such as certificates, text, and screenshots, continue to slip under the radar, leading to real-world risks and losses.
Despite the critical importance of detecting tampered document images, the availability of suitable datasets for researchers to develop and evaluate novel detection methodologies is currently limited. However, recent efforts have been made to address this issue. In 2020, the Phase V Security AI Challenger Program (SACP) organized the world's first adversarial attacks competition focused on document image forgery, the Forgery Detection on Certificate Image competition. The competition presented participants with a dataset of document images that closely resembled real-life scenarios and challenged them to develop effective forgery detection methods. Expanding on the success of the SACP competition, the organizers held the Real-World Image Forgery Localization Challenge (RIFLC) in 2022. This competition aimed to explore document image tamper detection in a more realistic environment and provided another dataset of document images for researchers to utilize. In 2023, a similar event, called the Detecting Tampered Text in Images (DTT) was held, and to facilitate diverse and comprehensive tampered text detection, the organizers constructed the Tampered Text in Images (TTI) dataset. This dataset replicates electronic commerce scenarios and contains images acquired from various sources. Besides the aforementioned three datasets, we also utilized our self-constructed dataset for supplementary and comparative analysis.
Table 1 presents the differences in quantity and characteristics of the four datasets, along with the proportion of tampered regions and other pertinent details.
\begin{enumerate}
\item SACP2020: In 2020, the Phase V Security AI Challenger Program (SACP) called Forgery Detection on Certificate Image is the first adversarial attacks competition in the world against document images forgery [18].
\item SACP2022: Subsequently, the organizers held a Read-World Image Forgery Localization Challenge to further study the document image tamper detection scheme in a more realistic environment in 2022 [19]. Both contests provide a forgery dataset of document images that approximates real-life scenarios.
\end{enumerate}
Texts in the image of the document are neatly lined up, and the structure lines are all straight. When forging images, it needs to think about whether the tampering boundary will destroy the content of the document. So, to make a fake document image that looks like the real thing, we have to let the tampering boundary fall on blank background areas instead of areas with structure contents.
Forgery postprocessing techniques include boundary blurring, contrast adjustment, and scaling, all of which are used to hide the obviousness of manipulation traces. Moreover, Chen[] et recommend that some post-processing (color pre-compensation and inverse halftoning) be used to reduce the visual artifacts of text editing operations while using the print-and-scan channel.
After that, we employ image blending to make sure the modified pixels are consistent with the target domain.
As shown in Fig. \ref{dataset_pic},
we firstly crawl over 8000 document images that had not been tampered with. Then two images are randomly picked from the document images, one image as source image \(A\) \(\in\) \(\mathbb{R}^{H_{A} \times W_{A} \times 3}\) and the other image as target image \(B\) \(\in\) \(\mathbb{R}^{H_{B} \times W_{B} \times 3}\). Then a mask \(M_{A}\) \(\in\) \(\mathbb{R}^{H_{A} \times W_{A} \times 1}\) is created for region of interest in the source image by a marking tool. Mask image \(M_{A}\) represents the binary image where 0 refers to the aim background should be preserved, while a 1 refers to the source object need to be replicated. According to Mask \(M_{A}\), the clipping region \(R_{A}\) \(\in\) \(\mathbb{R}^{H^{'}_{A} \times W^{'}_{A} \times 1}\) can be extracted from image \(A\), \emph{i.e.,}
\begin{equation}
R_{A} =A \odot M_{A}.
\label{eq27}
\end{equation}
A new naive forgery document image \(C\) \(\in\) \(\mathbb{R}^{H_{B} \times W_{B} \times 3}\) and a corresponding ground-truth mask image \(M_{C}\) \(\in\) \(\mathbb{R}^{H_{B} \times W_{B} \times 1}\) are generated by randomly splicing the clipping region \(R_{A}\) onto the target image \(B\). The mask \(M_{C}\) gives information about the region in the target image where the cropped area of source image has to be pasted,
\begin{equation}
C =R_{A}\odot M_{C} + B \odot\left(1-M_{C}\right).
\label{eq29}
\end{equation}
Then we use image blending to ensure the transformed pixels conform to the target domain to ensure consistency. Poisson image editing and Deep Image Blending are two fo the most popular methods for image blending \cite{Zhang2020DeepIB}. The idea of Poisson image editing is to reconstruct pixels in the blending region such that the blending boundary has smooth transitions or small gradients compared to the target image boundary pixels.
The method of Deep Image Blending is to jointly optimize the proposed Poisson blending loss as well as the style and content loss computed from a deep network. Finally, wee have produced \(18,526\) blending document images as our datasets. We also use the SACP dataset\cite{SACP2020} for a supplement and contrast.
By randomly selecting 70% and 30% of the images from the dataset, we divided both datasets into two subsets for training and testing.
\begin{figure}[t]\centering
\includegraphics[width=8.8cm]{imgs/s15.pdf}
\caption{Schematic diagram of MS-PEM. We used patches of different sizes for embedding features from three different scales to obtain embedded sequences of the same length. By fusing features from different stages, the information richness of the bottleneck is enhanced.}
\label{FIG0}
\end{figure}
As the semantic bottleneck acts as a connector among the four sub-networks, the global correspondence network \(\mathbf{\Phi_{\mathcal{G}}}\), estimation networks \(\mathbf{\Psi_{\mathcal{L}}}\) and \(\mathbf{\Psi_{\mathcal{G}}}\) aim to recover the information from the bottleneck. One way is to directly input the information from the last encoding layer of the feature extraction network \(\mathbf{\Phi_{\mathcal{L}}}\) into the bottleneck. However, this has a significant drawback: the last layer of the encoder network usually contains sparse semantic information, and it is difficult to recover low-level features from high-level representations directly. To overcome this challenge and enrich the information in the semantic bottleneck, we propose the method of multi-scale patch embedding.
As shown in Fig. 4, we utilized deep representations from the first to third stages of the encoder \(\mathbf{\Phi_{\mathcal{L}}}\) to generate informative patch embeddings for the semantic bottleneck. To ensure that the sequences obtained from different scales have the same length, we adopted different patch sizes (\(P=4,2,1\)) for different levels in the embedding stage. Finally, we added the sequences obtained from the three scales to obtain the final embedding with rich information.
\subsection{Main Results}
\textbf{Baseline Models} We present a comprehensive evaluation of our proposed model on four standard datasets, comparing its performance against state-of-the-art (SoTA) methods. Some methods are described below:
\begin{itemize}
%code
\item DFCN: PSCC-Net \footnote{\url{https://github.com/lihaod/Deep_inpainting_localization}} employs a progressive process to predict manipulation masks on multiple scales, utilizing each mask as prior information to aid in the prediction of subsequent-scale masks.
\item MVSS-Net: MVSS-Net\footnote{\url{https://github.com/lihaod/Deep_inpainting_localization}} addresses both the sensitivity in detecting tampered images and the specificity in real untampered images.
\item SE-Network: The proposed SE-Network\footnote{\url{https://github.com/lihaod/Deep_inpainting_localization}} introduces a multi-task squeeze and excitation network with two encoder-decoder streams. Leveraging image and mask edges, the model facilitates the learning of spliced masks.
\item RRU-Net: RRU-Net\footnote{\url{https://github.com/lihaod/Deep_inpainting_localization}} improves the learning capability of Convolutional Neural Networks (CNNs) by leveraging the propagation and feedback of CNN residuals.
\item CFL-Net: CFL-Net\footnote{\url{https://github.com/lihaod/Deep_inpainting_localization}} relies on the assumption that there exists a difference in feature distribution between untampered and manipulated regions in each forged image sample, regardless of the forgery type.
\end{itemize}
\textbf{Quantitative analysis.} The results are illustrated in Table \ref{Performance1}, which shows that our proposed model performs consistently better than other image forgery detection models.
Given the absence of an open source performance evaluation of the baseline model on the doctored document image benchmark dataset, we leverage the official open source code provided by the authors. To ensure consistency and comparability, we employ the same training configuration as outlined in our approach to implement these methods. Subsequently, we select the best-performing results from this comparative evaluation as our final outcomes. The experimental findings demonstrate that the proposed model exhibits commendable performance in terms of F1 and IOU metrics across all four datasets, with particularly notable performance observed on the Season3 dataset. This finding suggests that our model exhibits enhanced relevance in the context of tamper detection in text images.
We observed that the performance of the SE-Network is second to that of our model, indicating the necessity of the attention mechanism for tamper detection in text images. Furthermore, our Efficient Encoder naturally employs Se-Attention to extract tamper features.
The superiority of our model can likely be attributed to its utilization of high-scale semantic information acquired through a semantic segmentation network, along with multiple supervision modules. In contrast, previous approaches heavily relied on low-level clues as supplementary features.
\begin{figure}[!h]
\centering
\includegraphics[scale=0.36]{imgs/PRcurve.pdf}
\caption{Performance comparison with 12 state-of-the-art methods over 5 datasets. The first row shows comparison of precision-recall curves. The second row shows comparison of F-measure curves over different thresholds. As the figure shows, F\(^3\)Net achieves the best performance on all datasets.}
\label{PRcurve}
\end{figure}
\begin{table}[htb]
\caption{Performance comparison with 12 state-of-the-art methods over 5 datasets. MAE (smaller is better), mean Fmeasure (\(mF\), larger is better), Smeasure (\(S_\alpha\), larger is better) and Emeasure (\(E_\xi\), larger is better) are used to measure the model performance. The best results are highlighted in bold. Our model ranks first on all datasets and metrics.}
\label{Performance1}
\renewcommand\tabcolsep{2pt}
\renewcommand\arraystretch{2.0}
\centering
\small
\begin{tabular}{r|cccc|cccc|cccc|cccc|cccc}
\hline
\hline
\multirow{3}{}{\textbf{Algorithm}} & \multicolumn{4}{c|}{\textbf{SACP I}} & \multicolumn{4}{c|}{\textbf{SACP II}} & \multicolumn{4}{c|}{\textbf{SACP III}} & \multicolumn{4}{c|}{\textbf{PB}} & \multicolumn{4}{c}{\textbf{DocTamper}} \
& \multicolumn{4}{c|}{1,000 images} & \multicolumn{4}{c|}{850 images} & \multicolumn{4}{c|}{5,019 images} & \multicolumn{4}{c|}{4,447 images} & \multicolumn{4}{c}{5,168 images} \
& \(F_{1}\) & IoU & AUC & FNR & \(F_{1}\) & IoU & AUC & FNR & \(F_{1}\) & IoU & AUC & FNR & \(F_{1}\) & IoU & AUC & FNR& \(F_{1}\) & IoU & AUC & FNR \
\hline
\hline
DFCN\ref{DFCN_article} & .808 & .692 & .xx & .xx &
.653 & .519 & .xx & .xx &
.691 & .568 & .xx & .xx &
.643 & .624 & .xx & .xx &
.xx & .xx & .xx & .xx \
MVSS-Net\ref{MVSS} & .886 & .802 & .xx & .xx &
.797 & .683 & .xx & .xx &
.586 & .453 & .xx & .xx &
.588 & .531 & .xx & .xx &
.xx & .xx & .xx & .xx \
OSN-Net\ref{Wu2022RobustIF} & .xx & .xx & .xx & .xx &
.xx & .xx & .xx & .xx &
.xx & .xx & .xx & .xx &
.xx & .xx & .xx & .xx &
.xx & .xx & .xx & .xx \
SE-Network\ref{SE-Network} & .935 & .88 & .xx & .xx &
.688 & .56 & .xx & .xx &
.503 & .38 & .xx & .xx &
.632 & .608 & .xx & .xx &
.xx & .xx & .xx & .xx \
RRU-Net\ref{RRUNetTR} & .847 & .742 & .xx & .xx &
.301 & .204 & .xx & .xx &
.619 & .488 & .xx & .xx &
.658 & .602 & .xx & .xx &
.xx & .xx & .xx & .xx\
CFL-Net\ref{Niloy2022CFLNetIF} & .827 & .713 & .xx & .xx &
.838 & .733 & .xx & .xx &
.531 & .396 & .xx & .xx &
.526 & .463 & .xx & .xx &
.xx & .xx & .xx & .xx \
\hline
\textbf{F\(^3\)Net(ours)} &
\textbf{.921} & \textbf{.856} & \textbf{.xx} & \textbf{.xx} & %sea3
\textbf{.828} & \textbf{.726} & \textbf{.xx} & \textbf{.xx} & %sea5
\textbf{.728} & \textbf{.608} & \textbf{.xx} & \textbf{.xx} & %sea6
\textbf{.657} & \textbf{.637} & \textbf{.xx} & \textbf{.xx} & %pb
\textbf{.xx} & \textbf{.xx} & \textbf{.xx} & \textbf{.xx} \ %doctamper
\hline
\hline
\end{tabular}
\end{table*}
\textbf{Visual Comparison.}
%%%%%%%%%%%%%
The superior forgery localization performance of the proposed method is demonstrated through several visual examples in Fig. 8. This approach excels in accurately localizing even minute tampered regions and detecting minor modifications, such as alterations to individual digits. These capabilities are attributed to the fusion mechanism employed by the model, which integrates high-dimensional and low-dimensional features at multiple levels and scales. Additionally, our method not only effectively highlights salient object regions but also suppresses background noises, exhibiting excellent performance in various challenging scenarios. It is noteworthy that our method successfully identifies tampering regions across different types of manipulation, and the predictions are directly generated by our model without any post-processing.
%%%%%%%%%%%%% 泛化性
%As a result, on the two cross-domain subsets, they show bad cross-domain generalization ability, which is crucial in real-world document image tampering detection applications. The qualitative results for visual comparisons are illustrated in Fig.7
%%%%%%%%%%%%%
\begin{figure}[htbp]
\centering
\includegraphics[width=\textwidth]{imgs/Loc_results.pdf}
\caption{Examples of tampering localization results in different situations. Each
super-row (separated by dashed lines) corresponds to an example. In each
example, the distorted images \(\mathbf{I}^D\) from top to bottom were compressed with JPEG
quality factors 60, 70, and 80, respectively. Examples #1 and #2 are from
the Certificate PS dataset, examples #3 and #4 are from the DEFACTO
dataset, while the last example is from the IMD2020 dataset.}
\label{fig:LocalizationResults}
\end{figure}
\subsection{Ablation Studies}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Before analyzing the influence of each module, there are two hyper parameters ({\it i.e.}, $ \lambda_{1}$ and $ \lambda_{2}$) to be determined. \(\lambda_{1}\) is used in PPA loss to adjust the proportion of hard pixels. Tab.3 lists the scores of \(MAE\), \(mF\), \(S_\alpha\) and \(E_\xi\) when \(\gamma\) is given different values. As can be seen, when \(\gamma\) equals 5, these indicators reach highest scores. In addition, \(N\) represents the number of sub-decoders in CFD. We increase \(N\) gradually from 1 to 4 and measure the corresponding scores of above metrics, as shown in Tab. 4. When \(N\)=2, the model achieves the best performance. Both of these experiments are conducted on DUT-OMRON and DUTS.
\begin{table}[htb]
\label{gamma}
\centering
\caption{Comparison with different \(\gamma\). When \(\gamma=5\), the model achieves the best results.}
\renewcommand\tabcolsep{3.7pt}
\renewcommand\arraystretch{1.1}
\begin{tabular}{c|cccc|cccc}
\hline
& \multicolumn{4}{c|}{DUT-OMRON} & \multicolumn{4}{c}{DUTS-TE} \
& MAE & \(mF\) & \(S_\alpha\) & \(E_\xi\) & MAE & \(mF\) & \(S_\alpha\) & \(E_\xi\) \
\hline
\(\gamma\)=3 & .058 & .755 & .835 & .857 & .038 & .835 & .888 & .898 \
\(\gamma\)=4 & .057 & .758 & .837 & .859 & .037 & .837 & .888 & .900 \
\(\gamma\)=5 & .053 & .766 & .838 & .870 & .035 & .840 & .888 & .902 \
\(\gamma\)=6 & .060 & .752 & .833 & .855 & .038 & .834 & .887 & .897 \
\hline
\end{tabular}
\end{table}
We evaluate the necessity of all components of our proposed network framework by comparing the splice localization performance of several ablated versions of our model presented in Table \ref{tab_ablation} and Figure \ref{detection_results}. In this investigation, four aspects were considered: 1) effectiveness of the AFS module, 2) benefits of the ACF and BAL modules, 3) robustness to different attacks, and 4) qualitative visualization. In the following, we discuss these four aspects separately.
\textbf{Influence of the image classification loss.} To investigate the importance of different modules in F\(^3\)Net, we conduct a series of controlled experiments on DUTS, as shown in Tab.2. First, we test the effect of different loss functions, inlcuding BCE, IoU and PPA. Among them, PPA loss achieves the best performance on three evaluation metrics. Furthermore, we keep adding the multi-level supervision, cross feature module and cascaded feedback decoder to evaluate their performance. As we can see, all these modules boost the model performance. When these modules are combined, we can get the best SOD results. It demonstrates that all components are necessary for the proposed framework. The influence of the loss function on the performance of tampering localization was tested, and the results presented in Table I demonstrate that the combined loss approach outperforms the CE loss approach. This finding suggests that the combined loss function provides an advantage in localizing tampering.
\textbf{Influence of the image classification loss.} Comparing Seg+Clf and Seg, we see a clear increase in specificity and a clear drop in sensitivity, suggesting that adding lossclf makes the model more conservative for reporting manipulation. This change is not only confirmed by lower pixellevel performance, but is also observed in the fourth column of Fig. 6, showing that manipulated areas predicted by Seg+Clf are much reduced.
\textbf{Influence of ESB.} The better performance of Seg+Clf+E against Seg+Clf justifies the effectiveness of ESB. Seg+Clf+E/s is obtained by removing the Sobel operation from Seg+Clf+E, so its performance degeneration in particular on copy-move detection (from 0.405 to 0.382, cmpv in Table 3) indicates the necessity of this operation.
\textbf{Influence of the image classification loss.} Architecture analysis The self-attention mechanism is the key component for modeling rich interactions between set elements, where the positional encoding is very important. As shown in Tab. 3, performance is significantly improved only when self-attention and positional encoding are used at the same time. Dense correction improves the transparency of the hidden layers, which minimizes the loss error more effectively, and re-corrects the mask prediction based on semantic dependencies between different attention maps. From the last line in Tab. 3, we can observe that dense correction is helpful for the final performance.
%%%%%%%%%%%%%
\begin{table}[htb]
\centering
\caption{Pixel-level localization performance under the metrics of F1 and IoU comparison on each component of our proposed network}
\label{Ablation}
\renewcommand\tabcolsep{5pt}
\renewcommand\arraystretch{1.2}
\begin{tabular}{cccccc|cccc}
\hline
\multirow{2}{}{BCE} & \multirow{2}{}{DICE} & \multirow{2}{}{LSHE} & \multirow{2}{}{GAEM} & \multirow{2}{}{CFM} &\multirow{2}{}{TTA} & \multicolumn{4}{c}{Certi-Blending} \
& & & & & & \(F_{1}\) & \(IoU\) & \(FNR\) & \(AUC\) \
\hline
\checkmark & ~ & ~ & ~ & ~ & ~ & .051 & .779 & .861 & .871 \
~ & \checkmark & ~ & ~ & ~ & ~ & .047 & .783 & .864 & .874 \
\checkmark & \checkmark & ~ & ~ & ~ & ~ & .045 & .789 & .867 & .875 \
~ & ~ & \checkmark & ~ & ~ & ~ & .043 & .808 & .872 & .880 \
~ & ~ & \checkmark & \checkmark & ~ & ~ & .040 & .812 & .875 & .882 \
~ & ~ & \checkmark & \checkmark & \checkmark & ~ & .036 & .831 & .884 & .893 \
~ & ~ & \checkmark & \checkmark & \checkmark & \checkmark & .035 & .840 & .888 & .902 \
\hline
\end{tabular}
\end{table}
\subsection{Robustness Analysis}
\begin{figure}[htb]
\centering
\includegraphics[scale=0.52]{imgs/3distribution.pdf}
\caption{An overview of proposed F\(^3\)Net. ResNet-50 is used as the backbone encoder. Cross feature module (CFM) is used as the basic module to fuse features of different layers. Cascaded feedback decoder (CFD) contains multiple sub-decoders to feedback and refine multi-level features. Multi-level supervision (MLS) helps to ease the optimization of F\(^3\)Net.}
\label{framework}
\end{figure}
\begin{table}[t!]
\begin{center}
\renewcommand{\arraystretch}{1.4}
\caption{Robustness analysis of localization with respect to various distortions. Pixel-level AUCs are reported (in \(\%\)).}
\label{table:robustness}
\begin{adjustbox}{width=\linewidth}
\begin{tabular}{ccccccccccc}
\toprule
\multirow{2}{}{Distortion} & Resize & Resize & GSBlur & GSBlur & GSNoise & GSNoise & JPEGComp & JPEGComp & \multirow{2}{}{Mixed} & \multirow{2}{}{ w/o distortion} \ & \(0.78\times\) & \(0.25\times\) & \(k = 3\) & \(k = 15\) & \(\sigma = 3\) & \(\sigma = 15\) & \(q = 100\) & \(q = 50\) \ \hline
\multicolumn{11}{c}{Columbia} \ \hline
ManTraNet~\cite{wu2019mantra} & \(71.66\) & \(68.64\) & \(67.72\) & \(62.88\) & \(68.22\) & \(54.97\) & \(75.00\) & \(59.37\) & \(60.47\) & \(77.95\) \
SPAN~\cite{hu2020span} & \(89.99\) & \(69.08\) & \(78.97\) & \(67.70\) & \(75.11\) & \(65.80\) & \(93.32\) & \(74.62\) & \(62.54\) & \(93.60\) \
PSCC-Net & \(\mathbf{93.40}\) & \(\mathbf{78.41}\) & \(\mathbf{84.18}\) & \(\mathbf{73.24}\) & \(\mathbf{82.64}\) & \(\mathbf{74.35}\) & \(\mathbf{97.97}\) & \(\mathbf{89.11}\) & \(\mathbf{72.69}\) & \(\mathbf{98.19}\) \ \hline
\multicolumn{11}{c}{NIST16} \ \hline
ManTraNet~\cite{wu2019mantra} & \(77.43\) & \(75.52\) & \(77.46\) & \(74.55\) & \(67.41\) & \(58.55\) & \(77.91\) & \(74.38\) & \(64.82\) & \(78.05\) \
SPAN~\cite{hu2020span} & \(83.24\) & \(80.32\) & \(83.10\) & \(79.15\) & \(75.17\) & \(67.28\) & \(83.59\) & \(80.68\) & \(68.36\) & \(83.95\) \
PSCC-Net & \(\mathbf{85.29}\) & $ \mathbf{85.01}$ & \(\mathbf{85.38}\) & \(\mathbf{79.93}\) & \(\mathbf{78.42}\) & \(\mathbf{76.65}\) & \(\mathbf{85.40}\) & \(\mathbf{85.37}\) & \(\mathbf{73.93}\) & \(\mathbf{85.47}\) \
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
\end{table*}
%%%%%%%%%%%%%%%%%%
In this section, to further assess the robustness of the TBNet, different attacks including JPEG compression and scaling, are first used to the testing images from the CASIA1.0 and Carvalho datasets, respectively, and then the performance of TBNet is assessed on these two datasets. Their results are shown in Figs. 6 and 7. We note that in the scaling attacks, the scaling ratios of 0.7 and 0.5 are used in our experiments, and the JPEG compression consists of quality factors of 70 and 50. MCC CASIA and MCC carvalho indicate the MCC performance of the TBNet when the CASIA1.0 and Carvalho datasets are separately employed. F1 CASIA and F1 carvalho indicate the F1 performance of the TBNet when the CASIA1.0 and Carvalho datasets are separately used. It is observed from Fig. 6 that the MCC and F1 lines are essentially a straight line when the scaling attack is used so that the scaling attack has almost no effect on the TBNet. Similarly, when the JPEG compression attack is utilized in Fig. 7, the MCC and F1 lines show a slight downward trend, indicating that this attack degrades the performance of TBNet slightly, but it still can mine the remaining tampering traces from all of the channels. We think this is because some high-frequency information is lost in the jpeg compression process, and this information is very important for detecting the tampering image. Thus, these experimental results prove the stability of TBNet.
Following previous studies [3, 6, 11], we evaluate the model robustness against four image post-processing methods, Gaussian blur, Gaussian noise, JPEG compression and ISO noise over NIST dataset to verify the robustness of MSMGNet. The detailed results of robustness analysis are shown in Fig. 8. For each post-processing method, we vary the kernel size in Gaussian blur (from 3 to 9), variance of Gaussian noise (from 3 to 9), quality in JPEG compression (from 50 to 100), and variance of ISO noise (from 0.05 to 0.2) for comprehensive evaluation. As can be observed, Gaussian blur affects the detection performance more severely, in particular when a larger kernel size of 9×9, which blurs images and erases manipulation traces around tampered regions. In addition, compared with other baselines, MSMG-Net achieves the most general robust performance on ISO noise. The results of MSMG-Net is owed to our multi-scale multi-grained learning, where a parallel partial shunted transformer block designed to learn coarse-to-fine manipulation segmentation features. In summary, our model, MSMG-Net, consistently performs the best among all methods and can effectively tackle the challenges brought by various post-processing methods.
In this subsection, we evaluate the robustness of different methods by considering several common types of postprocessing. To this end, we enlarged the PS-scripted bookcover dataset by applying resizing, cropping, and noise adding to the generated tampered images with different factors and saving them with different JPEG qualities. Subsequently, we trained models with the enlarged dataset and performed testing on the PS-boundary and PS-arbitrary datasets. Three methods, i.e., Bayar’s 64 × 64, Forensic Similarity, and Mantra-net, were included for comparisons, since they perform relatively well in the experiments conducted in Section IV-C and IV-D.
As we all know, in real-world scenarios, different tampered images are usually subjected to different JPEG compressions. Training a specific ReLoc model for each JPEG compression under investigation is impractical because it would be timeconsuming and the implementations of JPEG compression would vary from manufacturers and software. Therefore, in this subsection, we evaluate the robustness against different JPEG compressions by using a single model.
The average improvements for QF 70 are relatively slighter, which are 0.042, 0.037, and 0.021 for DFCN, SCSE-Unet, and MVSS-net, respectively. When the testing QF is unseen in the training phase (i.e., QF 60), the average improvements achieved by ReLoc for DFCN, SCSE-Unet, and MVSS-net are 0.035, 0.010, and 0.012, respectively, which are still considerable. These experimental results indicate that ReLoc is also effective for improving the robustness against multiple JPEG compressions.
%%%%%%%%%%%%%%%%%%
\textbf{Experimental results under JPEG Compression Attack.} It is well-known that the JPEG compression process is lossy and leads to modified pixel values and information loss due to rounding errors. The comparative experimental results of the JPEG compression attack are presented in Figure 8(b). The results demonstrate that as the quality factor decreases from 100 to 50, the F1-Score of other deep learning detection methods experiences a significant decline, while ASGC-Net maintains its performance stability. Our module's JPEG artifacts removal capability can effectively mitigate the impact of JPEG compression, restoring and enhancing tampered region traces, which significantly improves the performance.
The comparative experiment results under different attacks are shown in Figure 7. Ordinates in Figure 7 represent the F1 score. From all subfigures in Figure 7, we can clearly see that the proposed method has the best localization performance under different attacks. Figure 7a is the result under JPEG compression. It can be observed that the slopes of all lines are very small, which indicates that these approaches are robust against JPEG compression with quality factors varying from 60 to 100. Figure 7b exhibits the performance under Gaussian noise. With the increase of standard deviation, the F1 scores of the proposed MFAN and EncNet [46] drop down gradually, and EncNet [46] degrades more rapidly than the proposed method.
\textbf{Experimental Results under Image Blending.} While image blending is a widely-used technique for post-processing tampered images, its robustness has not been comprehensively studied in recent research. To generate realistic tampered images, we employ image blending, which seamlessly merges overlapping areas from different images. This method ensures that the transformed pixels blend with the original image, obscuring boundaries and reducing color differences. Hence, it makes distinguishing between authentic and tampered images more challenging.
\textbf{Experimental results under scaling attack.} Scaling down an image rseduces its main features, making it challenging to identify tampering. Therefore, existing methods are often ineffective against scaling attacks, particularly image shrinking attacks. Specifically, we employ a range of scaling factors, varying from 0.6 to 1.4 with a step of 0.1, to resize the images.
The majority of images are vulnerable to unpredictability arising from content modifications or geometric distortions, such as compression, noise, and resizing. Therefore, it is essential for image tamper detection algorithms to consider the resilience of these alterations. To assess the effectiveness of our method on In-The-Wild and Carvalho datasets, we subjected the test images to various attacks, including JPEG compression, image scaling, and image blending.
The proposed method is capable of automatically adjusting the perceptual field size based on the shape of the tampered target, in addition to exhibiting significant resistance against external compression, noise, and scaling interference.
\begin{figure}[!h]
\centering
\includegraphics[scale=0.36]{imgs/noise.png}
\caption{Performance comparison with 12 state-of-the-art methods over 5 datasets. The first row shows comparison of precision-recall curves. The second row shows comparison of F-measure curves over different thresholds. As the figure shows, F\(^3\)Net achieves the best performance on all datasets.}
\label{PRcurve}
\end{figure}
\section{Discussion}
In our experiment, Data augmentation and pre-trained weights are employed to prevent overfilling and improve generalization. And experiments results also show that TTA can provide better segmentation results compared to only predicting on original images. We have also considered focal loss as a possible way to correct the imbalance between positive and negative samples, but the results are not ideal. The reason may be that the document image content is complex, it can be altered at any size, even as small as a letter.
\section{Conclusion}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Furthermore, we introduce a multi-scale estimation fusion mechanism to facilitate more effective detection results.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
As image editing tools and techniques continue to evolve, it is likely that new types of attacks will emerge that are even more difficult to detect. Text manipulation techniques can be used to alter the content of documents in a variety of ways, including the addition, deletion, or modification of text. xxx remains a significant challenge in the field of document image forensics. By continuing to explore new techniques and tools for detecting and localizing text tampering, researchers can help to mitigate the risks posed by these increasingly sophisticated technologies.
We construct a new dataset of counterfeit images to test the performance of the method we suggested for text image tamper detection and localization. This dataset contains examples of several sorts of text manipulation, such as text swapping, splicing, and removal. To boost the method's robustness, we preprocess the tamper image with advanced blending technology. These strategies can aid in the removal of visual discrepancies or artifacts, making the model more aware of manipulating details.
Overall, detecting text that has been erased or obliterated in text images is a complex problem that requires a combination of forensic analysis, OCR techniques, and deep learning-based methods.
\section{Acknowledgement}
This work was supported by the National Natural Science Foundation of China (61901237, 62171244), Alibaba Innovative Research Program.
% References
\bibliographystyle{ieeetr}
\bibliography{reference}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in, clip,keepaspectratio]{Yao.jpg}}]{Weipeng Liang} was born in Zhejiang, China, in 1998. He received the B.E. degree in the School of Media Engineering, Communication University of Zhejiang, Hangzhou, China, in 2020. He is currently pursuing the M.E. degree with the Faculty of Electrical Engineering and Computer Science, Ningbo University, Ningbo, China.
His research interests include information security and forensics.
\end{IEEEbiography}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Yu.jpg}}]{Wenyong Yu} received an M.S. degree and a Ph.D. degree from Huazhong University of Science and Technology, Wuhan, China, in 1999 and 2004, respectively. He is currently an Associate Professor with the School of Mechanical Science and Engineering, Huazhong University of Science and Technology.
His research interests include machine vision, intelligent control, and image processing.
\end{IEEEbiography}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{luowei.png}}]{Wei Luo}
will receive a B.S. degree from the School of Mechanical Science and Engineering, Huazhong University of Science and Technology, Wuhan, China, in 2023. He is gong to pursue a Ph.D. degree with the Department of Precision Instrument, Tsinghua University.
His research interests include deep learning, anomaly detection and machine vision.
\end{IEEEbiography}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{qzf.png}}]{Zhenfeng Qiang} received the B.S. degree in mechanical engineering from the Shaanxi University of Science and Technology, Xi’an, China, in 2017, and the M.S. degree in mechanical engineering from Jilin University (JLU), Changchun, China, in 2020.He is currently pursuing the Ph.D. degree with the Department of Precision Instrument, Tsinghua University, Beijing, China.
His research interests include system development of NDIR sensor, artificial intelligence, and biomechanics.
\end{IEEEbiography}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{ldh.png}}]{Donghao Luo} received the B.S. degree from the School of Intelligent System Engineering, Sun Yat-Sen University, Guangzhou, China in 2021. He is currently pursuing the Ph.D. degree with the Department of Precision Instrument, Tsinghua University.
His current research interests include smart grid and machine learning.
\end{IEEEbiography}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{zxt.png}}]{Xiaotian Zhang} received his B.S. degree in 2019 from Beihang University. Now he is a Ph.D. candidate with the Department of Precision Instrument, Tsinghua University.
His main research interests include anomaly detection, generative adversarial networks, and edge computing.
\end{IEEEbiography}
\vfill
\end{document}