summaryrefslogtreecommitdiff
path: root/fs
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-02-12 10:41:34 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2026-02-12 10:41:34 -0800
commit997f9640c9238b991b6c8abf5420b37bbba5d867 (patch)
tree511de4b17f8e0fe05d089f307c75449642be9f36 /fs
parent5903c871e21498405d11c5699d06becd12acda24 (diff)
parent433fbcac9ebe491b518b21c7305fba9a748c7d2c (diff)
Merge tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux
Pull fsverity updates from Eric Biggers: "fsverity cleanups, speedup, and memory usage optimization from Christoph Hellwig: - Move some logic into common code - Fix btrfs to reject truncates of fsverity files - Improve the readahead implementation - Store each inode's fsverity_info in a hash table instead of using a pointer in the filesystem-specific part of the inode. This optimizes for memory usage in the usual case where most files don't have fsverity enabled. - Look up the fsverity_info fewer times during verification, to amortize the hash table overhead" * tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux: fsverity: remove inode from fsverity_verification_ctx fsverity: use a hashtable to find the fsverity_info btrfs: consolidate fsverity_info lookup f2fs: consolidate fsverity_info lookup ext4: consolidate fsverity_info lookup fs: consolidate fsverity_info lookup in buffer.c fsverity: push out fsverity_info lookup fsverity: deconstify the inode pointer in struct fsverity_info fsverity: kick off hash readahead at data I/O submission time ext4: move ->read_folio and ->readahead to readpage.c readahead: push invalidate_lock out of page_cache_ra_unbounded fsverity: don't issue readahead for non-ENOENT errors from __filemap_get_folio fsverity: start consolidating pagecache code fsverity: pass struct file to ->write_merkle_tree_block f2fs: don't build the fsverity work handler for !CONFIG_FS_VERITY ext4: don't build the fsverity work handler for !CONFIG_FS_VERITY fs,fsverity: clear out fsverity_info from common code fs,fsverity: reject size changes on fsverity files in setattr_prepare
Diffstat (limited to 'fs')
-rw-r--r--fs/attr.c12
-rw-r--r--fs/btrfs/btrfs_inode.h4
-rw-r--r--fs/btrfs/extent_io.c53
-rw-r--r--fs/btrfs/inode.c13
-rw-r--r--fs/btrfs/verity.c11
-rw-r--r--fs/buffer.c25
-rw-r--r--fs/ext4/ext4.h8
-rw-r--r--fs/ext4/inode.c31
-rw-r--r--fs/ext4/readpage.c64
-rw-r--r--fs/ext4/super.c4
-rw-r--r--fs/ext4/verity.c34
-rw-r--r--fs/f2fs/compress.c7
-rw-r--r--fs/f2fs/data.c100
-rw-r--r--fs/f2fs/f2fs.h12
-rw-r--r--fs/f2fs/file.c6
-rw-r--r--fs/f2fs/inode.c1
-rw-r--r--fs/f2fs/super.c3
-rw-r--r--fs/f2fs/verity.c34
-rw-r--r--fs/inode.c9
-rw-r--r--fs/verity/Makefile1
-rw-r--r--fs/verity/enable.c41
-rw-r--r--fs/verity/fsverity_private.h20
-rw-r--r--fs/verity/open.c84
-rw-r--r--fs/verity/pagecache.c58
-rw-r--r--fs/verity/read_metadata.c19
-rw-r--r--fs/verity/verify.c91
26 files changed, 432 insertions, 313 deletions
diff --git a/fs/attr.c b/fs/attr.c
index b9ec6b47bab2..e7d7c6d19fe9 100644
--- a/fs/attr.c
+++ b/fs/attr.c
@@ -169,7 +169,17 @@ int setattr_prepare(struct mnt_idmap *idmap, struct dentry *dentry,
* ATTR_FORCE.
*/
if (ia_valid & ATTR_SIZE) {
- int error = inode_newsize_ok(inode, attr->ia_size);
+ int error;
+
+ /*
+ * Verity files are immutable, so deny truncates. This isn't
+ * covered by the open-time check because sys_truncate() takes a
+ * path, not an open file.
+ */
+ if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
+ return -EPERM;
+
+ error = inode_newsize_ok(inode, attr->ia_size);
if (error)
return error;
}
diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h
index 73602ee8de3f..55c272fe5d92 100644
--- a/fs/btrfs/btrfs_inode.h
+++ b/fs/btrfs/btrfs_inode.h
@@ -339,10 +339,6 @@ struct btrfs_inode {
struct rw_semaphore i_mmap_lock;
-#ifdef CONFIG_FS_VERITY
- struct fsverity_info *i_verity_info;
-#endif
-
struct inode vfs_inode;
};
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 3df399dc8856..744a1fff6eef 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -475,25 +475,25 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
end, page_ops);
}
-static bool btrfs_verify_folio(struct folio *folio, u64 start, u32 len)
+static bool btrfs_verify_folio(struct fsverity_info *vi, struct folio *folio,
+ u64 start, u32 len)
{
struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
- if (!fsverity_active(folio->mapping->host) ||
- btrfs_folio_test_uptodate(fs_info, folio, start, len) ||
- start >= i_size_read(folio->mapping->host))
+ if (!vi || btrfs_folio_test_uptodate(fs_info, folio, start, len))
return true;
- return fsverity_verify_folio(folio);
+ return fsverity_verify_folio(vi, folio);
}
-static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 len)
+static void end_folio_read(struct fsverity_info *vi, struct folio *folio,
+ bool uptodate, u64 start, u32 len)
{
struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
ASSERT(folio_pos(folio) <= start &&
start + len <= folio_next_pos(folio));
- if (uptodate && btrfs_verify_folio(folio, start, len))
+ if (uptodate && btrfs_verify_folio(vi, folio, start, len))
btrfs_folio_set_uptodate(fs_info, folio, start, len);
else
btrfs_folio_clear_uptodate(fs_info, folio, start, len);
@@ -573,14 +573,19 @@ static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio)
static void end_bbio_data_read(struct btrfs_bio *bbio)
{
struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
+ struct inode *inode = &bbio->inode->vfs_inode;
struct bio *bio = &bbio->bio;
+ struct fsverity_info *vi = NULL;
struct folio_iter fi;
ASSERT(!bio_flagged(bio, BIO_CLONED));
+
+ if (bbio->file_offset < i_size_read(inode))
+ vi = fsverity_get_info(inode);
+
bio_for_each_folio_all(fi, &bbio->bio) {
bool uptodate = !bio->bi_status;
struct folio *folio = fi.folio;
- struct inode *inode = folio->mapping->host;
u64 start = folio_pos(folio) + fi.offset;
btrfs_debug(fs_info,
@@ -615,7 +620,7 @@ static void end_bbio_data_read(struct btrfs_bio *bbio)
}
/* Update page status and unlock. */
- end_folio_read(folio, uptodate, start, fi.length);
+ end_folio_read(vi, folio, uptodate, start, fi.length);
}
bio_put(bio);
}
@@ -990,7 +995,8 @@ static void btrfs_readahead_expand(struct readahead_control *ractl,
* return 0 on success, otherwise return error
*/
static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
- struct btrfs_bio_ctrl *bio_ctrl)
+ struct btrfs_bio_ctrl *bio_ctrl,
+ struct fsverity_info *vi)
{
struct inode *inode = folio->mapping->host;
struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
@@ -1034,11 +1040,11 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
if (cur >= last_byte) {
folio_zero_range(folio, pg_offset, end - cur + 1);
- end_folio_read(folio, true, cur, end - cur + 1);
+ end_folio_read(vi, folio, true, cur, end - cur + 1);
break;
}
if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) {
- end_folio_read(folio, true, cur, blocksize);
+ end_folio_read(vi, folio, true, cur, blocksize);
continue;
}
/*
@@ -1050,7 +1056,7 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
*/
em = get_extent_map(BTRFS_I(inode), folio, cur, locked_end - cur + 1, em_cached);
if (IS_ERR(em)) {
- end_folio_read(folio, false, cur, end + 1 - cur);
+ end_folio_read(vi, folio, false, cur, end + 1 - cur);
return PTR_ERR(em);
}
extent_offset = cur - em->start;
@@ -1127,12 +1133,12 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
/* we've found a hole, just zero and go on */
if (block_start == EXTENT_MAP_HOLE) {
folio_zero_range(folio, pg_offset, blocksize);
- end_folio_read(folio, true, cur, blocksize);
+ end_folio_read(vi, folio, true, cur, blocksize);
continue;
}
/* the get_extent function already copied into the folio */
if (block_start == EXTENT_MAP_INLINE) {
- end_folio_read(folio, true, cur, blocksize);
+ end_folio_read(vi, folio, true, cur, blocksize);
continue;
}
@@ -1329,7 +1335,8 @@ again:
int btrfs_read_folio(struct file *file, struct folio *folio)
{
- struct btrfs_inode *inode = folio_to_inode(folio);
+ struct inode *vfs_inode = folio->mapping->host;
+ struct btrfs_inode *inode = BTRFS_I(vfs_inode);
const u64 start = folio_pos(folio);
const u64 end = start + folio_size(folio) - 1;
struct extent_state *cached_state = NULL;
@@ -1338,10 +1345,13 @@ int btrfs_read_folio(struct file *file, struct folio *folio)
.last_em_start = U64_MAX,
};
struct extent_map *em_cached = NULL;
+ struct fsverity_info *vi = NULL;
int ret;
lock_extents_for_read(inode, start, end, &cached_state);
- ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl);
+ if (folio_pos(folio) < i_size_read(vfs_inode))
+ vi = fsverity_get_info(vfs_inode);
+ ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi);
btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
btrfs_free_extent_map(em_cached);
@@ -2714,16 +2724,19 @@ void btrfs_readahead(struct readahead_control *rac)
.last_em_start = U64_MAX,
};
struct folio *folio;
- struct btrfs_inode *inode = BTRFS_I(rac->mapping->host);
+ struct inode *vfs_inode = rac->mapping->host;
+ struct btrfs_inode *inode = BTRFS_I(vfs_inode);
const u64 start = readahead_pos(rac);
const u64 end = start + readahead_length(rac) - 1;
struct extent_state *cached_state = NULL;
struct extent_map *em_cached = NULL;
+ struct fsverity_info *vi = NULL;
lock_extents_for_read(inode, start, end, &cached_state);
-
+ if (start < i_size_read(vfs_inode))
+ vi = fsverity_get_info(vfs_inode);
while ((folio = readahead_folio(rac)) != NULL)
- btrfs_do_readpage(folio, &em_cached, &bio_ctrl);
+ btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi);
btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 82df115bd0c5..845164485a00 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -34,7 +34,6 @@
#include <linux/sched/mm.h>
#include <linux/iomap.h>
#include <linux/unaligned.h>
-#include <linux/fsverity.h>
#include "misc.h"
#include "ctree.h"
#include "disk-io.h"
@@ -5616,11 +5615,8 @@ void btrfs_evict_inode(struct inode *inode)
trace_btrfs_inode_evict(inode);
- if (!root) {
- fsverity_cleanup_inode(inode);
- clear_inode(inode);
- return;
- }
+ if (!root)
+ goto clear_inode;
fs_info = inode_to_fs_info(inode);
evict_inode_truncate_pages(inode);
@@ -5720,7 +5716,7 @@ out:
* to retry these periodically in the future.
*/
btrfs_remove_delayed_node(BTRFS_I(inode));
- fsverity_cleanup_inode(inode);
+clear_inode:
clear_inode(inode);
}
@@ -8151,9 +8147,6 @@ static void init_once(void *foo)
struct btrfs_inode *ei = foo;
inode_init_once(&ei->vfs_inode);
-#ifdef CONFIG_FS_VERITY
- ei->i_verity_info = NULL;
-#endif
}
void __cold btrfs_destroy_cachep(void)
diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c
index 06cbd6f00a78..d12537a212e9 100644
--- a/fs/btrfs/verity.c
+++ b/fs/btrfs/verity.c
@@ -694,7 +694,6 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size)
*
* @inode: inode to read a merkle tree page for
* @index: page index relative to the start of the merkle tree
- * @num_ra_pages: number of pages to readahead. Optional, we ignore it
*
* The Merkle tree is stored in the filesystem btree, but its pages are cached
* with a logical position past EOF in the inode's mapping.
@@ -702,8 +701,7 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size)
* Returns the page we read, or an ERR_PTR on error.
*/
static struct page *btrfs_read_merkle_tree_page(struct inode *inode,
- pgoff_t index,
- unsigned long num_ra_pages)
+ pgoff_t index)
{
struct folio *folio;
u64 off = (u64)index << PAGE_SHIFT;
@@ -771,16 +769,17 @@ out:
/*
* fsverity op that writes a Merkle tree block into the btree.
*
- * @inode: inode to write a Merkle tree block for
+ * @file: file to write a Merkle tree block for
* @buf: Merkle tree block to write
* @pos: the position of the block in the Merkle tree (in bytes)
* @size: the Merkle tree block size (in bytes)
*
* Returns 0 on success or negative error code on failure
*/
-static int btrfs_write_merkle_tree_block(struct inode *inode, const void *buf,
+static int btrfs_write_merkle_tree_block(struct file *file, const void *buf,
u64 pos, unsigned int size)
{
+ struct inode *inode = file_inode(file);
loff_t merkle_pos = merkle_file_pos(inode);
if (merkle_pos < 0)
@@ -793,8 +792,6 @@ static int btrfs_write_merkle_tree_block(struct inode *inode, const void *buf,
}
const struct fsverity_operations btrfs_verityops = {
- .inode_info_offs = (int)offsetof(struct btrfs_inode, i_verity_info) -
- (int)offsetof(struct btrfs_inode, vfs_inode),
.begin_enable_verity = btrfs_begin_enable_verity,
.end_enable_verity = btrfs_end_enable_verity,
.get_verity_descriptor = btrfs_get_verity_descriptor,
diff --git a/fs/buffer.c b/fs/buffer.c
index b67791690ed3..ed724a902657 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -303,6 +303,7 @@ still_busy:
struct postprocess_bh_ctx {
struct work_struct work;
struct buffer_head *bh;
+ struct fsverity_info *vi;
};
static void verify_bh(struct work_struct *work)
@@ -312,21 +313,12 @@ static void verify_bh(struct work_struct *work)
struct buffer_head *bh = ctx->bh;
bool valid;
- valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh));
+ valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size,
+ bh_offset(bh));
end_buffer_async_read(bh, valid);
kfree(ctx);
}
-static bool need_fsverity(struct buffer_head *bh)
-{
- struct folio *folio = bh->b_folio;
- struct inode *inode = folio->mapping->host;
-
- return fsverity_active(inode) &&
- /* needed by ext4 */
- folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
-}
-
static void decrypt_bh(struct work_struct *work)
{
struct postprocess_bh_ctx *ctx =
@@ -336,7 +328,7 @@ static void decrypt_bh(struct work_struct *work)
err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size,
bh_offset(bh));
- if (err == 0 && need_fsverity(bh)) {
+ if (err == 0 && ctx->vi) {
/*
* We use different work queues for decryption and for verity
* because verity may require reading metadata pages that need
@@ -358,15 +350,20 @@ static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
{
struct inode *inode = bh->b_folio->mapping->host;
bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode);
- bool verify = need_fsverity(bh);
+ struct fsverity_info *vi = NULL;
+
+ /* needed by ext4 */
+ if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
+ vi = fsverity_get_info(inode);
/* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */
- if (uptodate && (decrypt || verify)) {
+ if (uptodate && (decrypt || vi)) {
struct postprocess_bh_ctx *ctx =
kmalloc(sizeof(*ctx), GFP_ATOMIC);
if (ctx) {
ctx->bh = bh;
+ ctx->vi = vi;
if (decrypt) {
INIT_WORK(&ctx->work, decrypt_bh);
fscrypt_enqueue_decrypt_work(&ctx->work);
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index e0ed273e2e8a..293f698b7042 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -1196,10 +1196,6 @@ struct ext4_inode_info {
#ifdef CONFIG_FS_ENCRYPTION
struct fscrypt_inode_info *i_crypt_info;
#endif
-
-#ifdef CONFIG_FS_VERITY
- struct fsverity_info *i_verity_info;
-#endif
};
/*
@@ -3744,8 +3740,8 @@ static inline void ext4_set_de_type(struct super_block *sb,
}
/* readpages.c */
-extern int ext4_mpage_readpages(struct inode *inode,
- struct readahead_control *rac, struct folio *folio);
+int ext4_read_folio(struct file *file, struct folio *folio);
+void ext4_readahead(struct readahead_control *rac);
extern int __init ext4_init_post_read_processing(void);
extern void ext4_exit_post_read_processing(void);
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 15ba4d42982f..8a544f79dcb4 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -3373,33 +3373,6 @@ out:
return ret;
}
-static int ext4_read_folio(struct file *file, struct folio *folio)
-{
- int ret = -EAGAIN;
- struct inode *inode = folio->mapping->host;
-
- trace_ext4_read_folio(inode, folio);
-
- if (ext4_has_inline_data(inode))
- ret = ext4_readpage_inline(inode, folio);
-
- if (ret == -EAGAIN)
- return ext4_mpage_readpages(inode, NULL, folio);
-
- return ret;
-}
-
-static void ext4_readahead(struct readahead_control *rac)
-{
- struct inode *inode = rac->mapping->host;
-
- /* If the file has inline data, no need to do readahead. */
- if (ext4_has_inline_data(inode))
- return;
-
- ext4_mpage_readpages(inode, rac, NULL);
-}
-
static void ext4_invalidate_folio(struct folio *folio, size_t offset,
size_t length)
{
@@ -5815,10 +5788,6 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
if (error)
return error;
- error = fsverity_prepare_setattr(dentry, attr);
- if (error)
- return error;
-
if (is_quota_modification(idmap, inode, attr)) {
error = dquot_initialize(inode);
if (error)
diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
index 49a6d36a8dba..830f3b8a321f 100644
--- a/fs/ext4/readpage.c
+++ b/fs/ext4/readpage.c
@@ -46,6 +46,7 @@
#include <linux/pagevec.h>
#include "ext4.h"
+#include <trace/events/ext4.h>
#define NUM_PREALLOC_POST_READ_CTXS 128
@@ -62,6 +63,7 @@ enum bio_post_read_step {
struct bio_post_read_ctx {
struct bio *bio;
+ struct fsverity_info *vi;
struct work_struct work;
unsigned int cur_step;
unsigned int enabled_steps;
@@ -97,6 +99,7 @@ static void verity_work(struct work_struct *work)
struct bio_post_read_ctx *ctx =
container_of(work, struct bio_post_read_ctx, work);
struct bio *bio = ctx->bio;
+ struct fsverity_info *vi = ctx->vi;
/*
* fsverity_verify_bio() may call readahead() again, and although verity
@@ -109,7 +112,7 @@ static void verity_work(struct work_struct *work)
mempool_free(ctx, bio_post_read_ctx_pool);
bio->bi_private = NULL;
- fsverity_verify_bio(bio);
+ fsverity_verify_bio(vi, bio);
__read_end_io(bio);
}
@@ -131,7 +134,8 @@ static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
ctx->cur_step++;
fallthrough;
case STEP_VERITY:
- if (ctx->enabled_steps & (1 << STEP_VERITY)) {
+ if (IS_ENABLED(CONFIG_FS_VERITY) &&
+ ctx->enabled_steps & (1 << STEP_VERITY)) {
INIT_WORK(&ctx->work, verity_work);
fsverity_enqueue_verify_work(&ctx->work);
return;
@@ -172,22 +176,16 @@ static void mpage_end_io(struct bio *bio)
__read_end_io(bio);
}
-static inline bool ext4_need_verity(const struct inode *inode, pgoff_t idx)
-{
- return fsverity_active(inode) &&
- idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
-}
-
static void ext4_set_bio_post_read_ctx(struct bio *bio,
const struct inode *inode,
- pgoff_t first_idx)
+ struct fsverity_info *vi)
{
unsigned int post_read_steps = 0;
if (fscrypt_inode_uses_fs_layer_crypto(inode))
post_read_steps |= 1 << STEP_DECRYPT;
- if (ext4_need_verity(inode, first_idx))
+ if (vi)
post_read_steps |= 1 << STEP_VERITY;
if (post_read_steps) {
@@ -196,6 +194,7 @@ static void ext4_set_bio_post_read_ctx(struct bio *bio,
mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
ctx->bio = bio;
+ ctx->vi = vi;
ctx->enabled_steps = post_read_steps;
bio->bi_private = ctx;
}
@@ -209,7 +208,7 @@ static inline loff_t ext4_readpage_limit(struct inode *inode)
return i_size_read(inode);
}
-int ext4_mpage_readpages(struct inode *inode,
+static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
struct readahead_control *rac, struct folio *folio)
{
struct bio *bio = NULL;
@@ -329,8 +328,7 @@ int ext4_mpage_readpages(struct inode *inode,
folio_zero_segment(folio, first_hole << blkbits,
folio_size(folio));
if (first_hole == 0) {
- if (ext4_need_verity(inode, folio->index) &&
- !fsverity_verify_folio(folio))
+ if (vi && !fsverity_verify_folio(vi, folio))
goto set_error_page;
folio_end_read(folio, true);
continue;
@@ -358,7 +356,7 @@ int ext4_mpage_readpages(struct inode *inode,
REQ_OP_READ, GFP_KERNEL);
fscrypt_set_bio_crypt_ctx(bio, inode, next_block,
GFP_KERNEL);
- ext4_set_bio_post_read_ctx(bio, inode, folio->index);
+ ext4_set_bio_post_read_ctx(bio, inode, vi);
bio->bi_iter.bi_sector = first_block << (blkbits - 9);
bio->bi_end_io = mpage_end_io;
if (rac)
@@ -394,6 +392,44 @@ next_page:
return 0;
}
+int ext4_read_folio(struct file *file, struct folio *folio)
+{
+ struct inode *inode = folio->mapping->host;
+ struct fsverity_info *vi = NULL;
+ int ret;
+
+ trace_ext4_read_folio(inode, folio);
+
+ if (ext4_has_inline_data(inode)) {
+ ret = ext4_readpage_inline(inode, folio);
+ if (ret != -EAGAIN)
+ return ret;
+ }
+
+ if (folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
+ vi = fsverity_get_info(inode);
+ if (vi)
+ fsverity_readahead(vi, folio->index, folio_nr_pages(folio));
+ return ext4_mpage_readpages(inode, vi, NULL, folio);
+}
+
+void ext4_readahead(struct readahead_control *rac)
+{
+ struct inode *inode = rac->mapping->host;
+ struct fsverity_info *vi = NULL;
+
+ /* If the file has inline data, no need to do readahead. */
+ if (ext4_has_inline_data(inode))
+ return;
+
+ if (readahead_index(rac) < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
+ vi = fsverity_get_info(inode);
+ if (vi)
+ fsverity_readahead(vi, readahead_index(rac),
+ readahead_count(rac));
+ ext4_mpage_readpages(inode, vi, rac, NULL);
+}
+
int __init ext4_init_post_read_processing(void)
{
bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, SLAB_RECLAIM_ACCOUNT);
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 69eb63dde983..504148b2142b 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -1489,9 +1489,6 @@ static void init_once(void *foo)
#ifdef CONFIG_FS_ENCRYPTION
ei->i_crypt_info = NULL;
#endif
-#ifdef CONFIG_FS_VERITY
- ei->i_verity_info = NULL;
-#endif
}
static int __init init_inodecache(void)
@@ -1539,7 +1536,6 @@ void ext4_clear_inode(struct inode *inode)
EXT4_I(inode)->jinode = NULL;
}
fscrypt_put_encryption_info(inode);
- fsverity_cleanup_inode(inode);
}
static struct inode *ext4_nfs_get_inode(struct super_block *sb,
diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c
index 667f9e8d4da9..ca61da53f313 100644
--- a/fs/ext4/verity.c
+++ b/fs/ext4/verity.c
@@ -360,42 +360,32 @@ static int ext4_get_verity_descriptor(struct inode *inode, void *buf,
}
static struct page *ext4_read_merkle_tree_page(struct inode *inode,
- pgoff_t index,
- unsigned long num_ra_pages)
+ pgoff_t index)
{
- struct folio *folio;
-
index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;
+ return generic_read_merkle_tree_page(inode, index);
+}
- folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0);
- if (IS_ERR(folio) || !folio_test_uptodate(folio)) {
- DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
-
- if (!IS_ERR(folio))
- folio_put(folio);
- else if (num_ra_pages > 1)
- page_cache_ra_unbounded(&ractl, num_ra_pages, 0);
- folio = read_mapping_folio(inode->i_mapping, index, NULL);
- if (IS_ERR(folio))
- return ERR_CAST(folio);
- }
- return folio_file_page(folio, index);
+static void ext4_readahead_merkle_tree(struct inode *inode, pgoff_t index,
+ unsigned long nr_pages)
+{
+ index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;
+ generic_readahead_merkle_tree(inode, index, nr_pages);
}
-static int ext4_write_merkle_tree_block(struct inode *inode, const void *buf,
+static int ext4_write_merkle_tree_block(struct file *file, const void *buf,
u64 pos, unsigned int size)
{
- pos += ext4_verity_metadata_pos(inode);
+ pos += ext4_verity_metadata_pos(file_inode(file));
- return pagecache_write(inode, buf, size, pos);
+ return pagecache_write(file_inode(file), buf, size, pos);
}
const struct fsverity_operations ext4_verityops = {
- .inode_info_offs = (int)offsetof(struct ext4_inode_info, i_verity_info) -
- (int)offsetof(struct ext4_inode_info, vfs_inode),
.begin_enable_verity = ext4_begin_enable_verity,
.end_enable_verity = ext4_end_enable_verity,
.get_verity_descriptor = ext4_get_verity_descriptor,
.read_merkle_tree_page = ext4_read_merkle_tree_page,
+ .readahead_merkle_tree = ext4_readahead_merkle_tree,
.write_merkle_tree_block = ext4_write_merkle_tree_block,
};
diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
index 7b68bf22989d..ef1225af2acf 100644
--- a/fs/f2fs/compress.c
+++ b/fs/f2fs/compress.c
@@ -1181,6 +1181,7 @@ int f2fs_prepare_compress_overwrite(struct inode *inode,
.cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
.rpages = NULL,
.nr_rpages = 0,
+ .vi = NULL, /* can't write to fsverity files */
};
return prepare_compress_overwrite(&cc, pagep, index, fsdata);
@@ -1716,7 +1717,7 @@ struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)
dic->nr_cpages = cc->nr_cpages;
refcount_set(&dic->refcnt, 1);
dic->failed = false;
- dic->need_verity = f2fs_need_verity(cc->inode, start_idx);
+ dic->vi = cc->vi;
for (i = 0; i < dic->cluster_size; i++)
dic->rpages[i] = cc->rpages[i];
@@ -1814,7 +1815,7 @@ static void f2fs_verify_cluster(struct work_struct *work)
if (!rpage)
continue;
- if (fsverity_verify_page(rpage))
+ if (fsverity_verify_page(dic->vi, rpage))
SetPageUptodate(rpage);
else
ClearPageUptodate(rpage);
@@ -1833,7 +1834,7 @@ void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
{
int i;
- if (!failed && dic->need_verity) {
+ if (IS_ENABLED(CONFIG_FS_VERITY) && !failed && dic->vi) {
/*
* Note that to avoid deadlocks, the verity work can't be done
* on the decompression workqueue. This is because verifying
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index c3dd8a5c8589..491f66511201 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -109,6 +109,7 @@ enum bio_post_read_step {
struct bio_post_read_ctx {
struct bio *bio;
struct f2fs_sb_info *sbi;
+ struct fsverity_info *vi;
struct work_struct work;
unsigned int enabled_steps;
/*
@@ -165,6 +166,7 @@ static void f2fs_verify_bio(struct work_struct *work)
container_of(work, struct bio_post_read_ctx, work);
struct bio *bio = ctx->bio;
bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS);
+ struct fsverity_info *vi = ctx->vi;
/*
* fsverity_verify_bio() may call readahead() again, and while verity
@@ -187,13 +189,13 @@ static void f2fs_verify_bio(struct work_struct *work)
struct folio *folio = fi.folio;
if (!f2fs_is_compressed_page(folio) &&
- !fsverity_verify_page(&folio->page)) {
+ !fsverity_verify_page(vi, &folio->page)) {
bio->bi_status = BLK_STS_IOERR;
break;
}
}
} else {
- fsverity_verify_bio(bio);
+ fsverity_verify_bio(vi, bio);
}
f2fs_finish_read_bio(bio, true);
@@ -1036,7 +1038,8 @@ out:
f2fs_up_write(&io->io_rwsem);
}
-static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
+static struct bio *f2fs_grab_read_bio(struct inode *inode,
+ struct fsverity_info *vi, block_t blkaddr,
unsigned nr_pages, blk_opf_t op_flag,
pgoff_t first_idx, bool for_write)
{
@@ -1057,7 +1060,7 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
if (fscrypt_inode_uses_fs_layer_crypto(inode))
post_read_steps |= STEP_DECRYPT;
- if (f2fs_need_verity(inode, first_idx))
+ if (vi)
post_read_steps |= STEP_VERITY;
/*
@@ -1072,6 +1075,7 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
ctx->bio = bio;
ctx->sbi = sbi;
+ ctx->vi = vi;
ctx->enabled_steps = post_read_steps;
ctx->fs_blkaddr = blkaddr;
ctx->decompression_attempted = false;
@@ -1083,15 +1087,15 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
}
/* This can handle encryption stuffs */
-static void f2fs_submit_page_read(struct inode *inode, struct folio *folio,
- block_t blkaddr, blk_opf_t op_flags,
- bool for_write)
+static void f2fs_submit_page_read(struct inode *inode, struct fsverity_info *vi,
+ struct folio *folio, block_t blkaddr,
+ blk_opf_t op_flags, bool for_write)
{
struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
struct bio *bio;
- bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags,
- folio->index, for_write);
+ bio = f2fs_grab_read_bio(inode, vi, blkaddr, 1, op_flags, folio->index,
+ for_write);
/* wait for GCed page writeback via META_MAPPING */
f2fs_wait_on_block_writeback(inode, blkaddr);
@@ -1193,6 +1197,14 @@ int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
return err;
}
+static inline struct fsverity_info *f2fs_need_verity(const struct inode *inode,
+ pgoff_t idx)
+{
+ if (idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
+ return fsverity_get_info(inode);
+ return NULL;
+}
+
struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index,
blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs)
{
@@ -1258,8 +1270,8 @@ got_it:
return folio;
}
- f2fs_submit_page_read(inode, folio, dn.data_blkaddr,
- op_flags, for_write);
+ f2fs_submit_page_read(inode, f2fs_need_verity(inode, folio->index),
+ folio, dn.data_blkaddr, op_flags, for_write);
return folio;
put_err:
@@ -2063,12 +2075,12 @@ static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac)
return rac ? REQ_RAHEAD : 0;
}
-static int f2fs_read_single_page(struct inode *inode, struct folio *folio,
- unsigned nr_pages,
- struct f2fs_map_blocks *map,
- struct bio **bio_ret,
- sector_t *last_block_in_bio,
- struct readahead_control *rac)
+static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi,
+ struct folio *folio, unsigned int nr_pages,
+ struct f2fs_map_blocks *map,
+ struct bio **bio_ret,
+ sector_t *last_block_in_bio,
+ struct readahead_control *rac)
{
struct bio *bio = *bio_ret;
const unsigned int blocksize = F2FS_BLKSIZE;
@@ -2120,8 +2132,7 @@ got_it:
} else {
zero_out:
folio_zero_segment(folio, 0, folio_size(folio));
- if (f2fs_need_verity(inode, index) &&
- !fsverity_verify_folio(folio)) {
+ if (vi && !fsverity_verify_folio(vi, folio)) {
ret = -EIO;
goto out;
}
@@ -2143,9 +2154,8 @@ submit_and_realloc:
bio = NULL;
}
if (bio == NULL)
- bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
- f2fs_ra_op_flags(rac), index,
- false);
+ bio = f2fs_grab_read_bio(inode, vi, block_nr, nr_pages,
+ f2fs_ra_op_flags(rac), index, false);
/*
* If the page is under writeback, we need to wait for
@@ -2295,9 +2305,10 @@ submit_and_realloc:
}
if (!bio)
- bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages - i,
- f2fs_ra_op_flags(rac),
- folio->index, for_write);
+ bio = f2fs_grab_read_bio(inode, cc->vi, blkaddr,
+ nr_pages - i,
+ f2fs_ra_op_flags(rac),
+ folio->index, for_write);
if (!bio_add_folio(bio, folio, blocksize, 0))
goto submit_and_realloc;
@@ -2336,7 +2347,7 @@ out:
* This function was originally taken from fs/mpage.c, and customized for f2fs.
* Major change was from block_size == page_size in f2fs by default.
*/
-static int f2fs_mpage_readpages(struct inode *inode,
+static int f2fs_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
struct readahead_control *rac, struct folio *folio)
{
struct bio *bio = NULL;
@@ -2391,6 +2402,7 @@ static int f2fs_mpage_readpages(struct inode *inode,
/* there are remained compressed pages, submit them */
if (!f2fs_cluster_can_merge_page(&cc, index)) {
+ cc.vi = vi;
ret = f2fs_read_multi_pages(&cc, &bio,
max_nr_pages,
&last_block_in_bio,
@@ -2424,8 +2436,9 @@ static int f2fs_mpage_readpages(struct inode *inode,
read_single_page:
#endif
- ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map,
- &bio, &last_block_in_bio, rac);
+ ret = f2fs_read_single_page(inode, vi, folio, max_nr_pages,
+ &map, &bio, &last_block_in_bio,
+ rac);
if (ret) {
#ifdef CONFIG_F2FS_FS_COMPRESSION
set_error_page:
@@ -2441,6 +2454,7 @@ next_page:
if (f2fs_compressed_file(inode)) {
/* last page */
if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) {
+ cc.vi = vi;
ret = f2fs_read_multi_pages(&cc, &bio,
max_nr_pages,
&last_block_in_bio,
@@ -2458,7 +2472,8 @@ next_page:
static int f2fs_read_data_folio(struct file *file, struct folio *folio)
{
struct inode *inode = folio->mapping->host;
- int ret = -EAGAIN;
+ struct fsverity_info *vi = NULL;
+ int ret;
trace_f2fs_readpage(folio, DATA);
@@ -2468,16 +2483,22 @@ static int f2fs_read_data_folio(struct file *file, struct folio *folio)
}
/* If the file has inline data, try to read it directly */
- if (f2fs_has_inline_data(inode))
+ if (f2fs_has_inline_data(inode)) {
ret = f2fs_read_inline_data(inode, folio);
- if (ret == -EAGAIN)
- ret = f2fs_mpage_readpages(inode, NULL, folio);
- return ret;
+ if (ret != -EAGAIN)
+ return ret;
+ }
+
+ vi = f2fs_need_verity(inode, folio->index);
+ if (vi)
+ fsverity_readahead(vi, folio->index, folio_nr_pages(folio));
+ return f2fs_mpage_readpages(inode, vi, NULL, folio);
}
static void f2fs_readahead(struct readahead_control *rac)
{
struct inode *inode = rac->mapping->host;
+ struct fsverity_info *vi = NULL;
trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac));
@@ -2488,7 +2509,11 @@ static void f2fs_readahead(struct readahead_control *rac)
if (f2fs_has_inline_data(inode))
return;
- f2fs_mpage_readpages(inode, rac, NULL);
+ vi = f2fs_need_verity(inode, readahead_index(rac));
+ if (vi)
+ fsverity_readahead(vi, readahead_index(rac),
+ readahead_count(rac));
+ f2fs_mpage_readpages(inode, vi, rac, NULL);
}
int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
@@ -3637,9 +3662,10 @@ repeat:
err = -EFSCORRUPTED;
goto put_folio;
}
- f2fs_submit_page_read(use_cow ?
- F2FS_I(inode)->cow_inode : inode,
- folio, blkaddr, 0, true);
+ f2fs_submit_page_read(use_cow ? F2FS_I(inode)->cow_inode :
+ inode,
+ NULL, /* can't write to fsverity files */
+ folio, blkaddr, 0, true);
folio_lock(folio);
if (unlikely(folio->mapping != mapping)) {
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 9f3aa3c7f126..a90a62cfe617 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -974,9 +974,6 @@ struct f2fs_inode_info {
#ifdef CONFIG_FS_ENCRYPTION
struct fscrypt_inode_info *i_crypt_info; /* filesystem encryption info */
#endif
-#ifdef CONFIG_FS_VERITY
- struct fsverity_info *i_verity_info; /* filesystem verity info */
-#endif
};
static inline void get_read_extent_info(struct extent_info *ext,
@@ -1603,6 +1600,7 @@ struct compress_ctx {
size_t clen; /* valid data length in cbuf */
void *private; /* payload buffer for specified compression algorithm */
void *private2; /* extra payload buffer */
+ struct fsverity_info *vi; /* verity info if needed */
};
/* compress context for write IO path */
@@ -1658,7 +1656,7 @@ struct decompress_io_ctx {
refcount_t refcnt;
bool failed; /* IO error occurred before decompression? */
- bool need_verity; /* need fs-verity verification after decompression? */
+ struct fsverity_info *vi; /* fs-verity context if needed */
unsigned char compress_algorithm; /* backup algorithm type */
void *private; /* payload buffer for specified decompression algorithm */
void *private2; /* extra payload buffer */
@@ -4886,12 +4884,6 @@ static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi,
return sbi->aligned_blksize;
}
-static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
-{
- return fsverity_active(inode) &&
- idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
-}
-
#ifdef CONFIG_F2FS_FAULT_INJECTION
extern int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate,
unsigned long type, enum fault_option fo);
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index 31a0c1bb41f6..1fdbe18692be 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -1076,10 +1076,6 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
if (err)
return err;
- err = fsverity_prepare_setattr(dentry, attr);
- if (err)
- return err;
-
if (unlikely(IS_IMMUTABLE(inode)))
return -EPERM;
@@ -4424,7 +4420,9 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len)
pgoff_t redirty_idx = page_idx;
int page_len = 0, ret = 0;
+ filemap_invalidate_lock_shared(mapping);
page_cache_ra_unbounded(&ractl, len, 0);
+ filemap_invalidate_unlock_shared(mapping);
do {
folio = read_cache_folio(mapping, page_idx, NULL, NULL);
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index 38b8994bc1b2..ee332b994348 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -1000,7 +1000,6 @@ no_delete:
}
out_clear:
fscrypt_put_encryption_info(inode);
- fsverity_cleanup_inode(inode);
clear_inode(inode);
}
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index c4c225e09dc4..cd00d030edda 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -504,9 +504,6 @@ static void init_once(void *foo)
#ifdef CONFIG_FS_ENCRYPTION
fi->i_crypt_info = NULL;
#endif
-#ifdef CONFIG_FS_VERITY
- fi->i_verity_info = NULL;
-#endif
}
#ifdef CONFIG_QUOTA
diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c
index 05b935b55216..92ebcc19cab0 100644
--- a/fs/f2fs/verity.c
+++ b/fs/f2fs/verity.c
@@ -256,42 +256,32 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf,
}
static struct page *f2fs_read_merkle_tree_page(struct inode *inode,
- pgoff_t index,
- unsigned long num_ra_pages)
+ pgoff_t index)
{
- struct folio *folio;
-
index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;
+ return generic_read_merkle_tree_page(inode, index);
+}
- folio = f2fs_filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0);
- if (IS_ERR(folio) || !folio_test_uptodate(folio)) {
- DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
-
- if (!IS_ERR(folio))
- folio_put(folio);
- else if (num_ra_pages > 1)
- page_cache_ra_unbounded(&ractl, num_ra_pages, 0);
- folio = read_mapping_folio(inode->i_mapping, index, NULL);
- if (IS_ERR(folio))
- return ERR_CAST(folio);
- }
- return folio_file_page(folio, index);
+static void f2fs_readahead_merkle_tree(struct inode *inode, pgoff_t index,
+ unsigned long nr_pages)
+{
+ index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;
+ generic_readahead_merkle_tree(inode, index, nr_pages);
}
-static int f2fs_write_merkle_tree_block(struct inode *inode, const void *buf,
+static int f2fs_write_merkle_tree_block(struct file *file, const void *buf,
u64 pos, unsigned int size)
{
- pos += f2fs_verity_metadata_pos(inode);
+ pos += f2fs_verity_metadata_pos(file_inode(file));
- return pagecache_write(inode, buf, size, pos);
+ return pagecache_write(file_inode(file), buf, size, pos);
}
const struct fsverity_operations f2fs_verityops = {
- .inode_info_offs = (int)offsetof(struct f2fs_inode_info, i_verity_info) -
- (int)offsetof(struct f2fs_inode_info, vfs_inode),
.begin_enable_verity = f2fs_begin_enable_verity,
.end_enable_verity = f2fs_end_enable_verity,
.get_verity_descriptor = f2fs_get_verity_descriptor,
.read_merkle_tree_page = f2fs_read_merkle_tree_page,
+ .readahead_merkle_tree = f2fs_readahead_merkle_tree,
.write_merkle_tree_block = f2fs_write_merkle_tree_block,
};
diff --git a/fs/inode.c b/fs/inode.c
index dae43a8de7e0..cc12b68e021b 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -14,6 +14,7 @@
#include <linux/cdev.h>
#include <linux/memblock.h>
#include <linux/fsnotify.h>
+#include <linux/fsverity.h>
#include <linux/mount.h>
#include <linux/posix_acl.h>
#include <linux/buffer_head.h> /* for inode_has_buffers */
@@ -774,6 +775,14 @@ void dump_mapping(const struct address_space *mapping)
void clear_inode(struct inode *inode)
{
/*
+ * Only IS_VERITY() inodes can have verity info, so start by checking
+ * for IS_VERITY() (which is faster than retrieving the pointer to the
+ * verity info). This minimizes overhead for non-verity inodes.
+ */
+ if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
+ fsverity_cleanup_inode(inode);
+
+ /*
* We have to cycle the i_pages lock here because reclaim can be in the
* process of removing the last page (in __filemap_remove_folio())
* and we must not free the mapping under it.
diff --git a/fs/verity/Makefile b/fs/verity/Makefile
index 435559a4fa9e..ddb4a88a0d60 100644
--- a/fs/verity/Makefile
+++ b/fs/verity/Makefile
@@ -5,6 +5,7 @@ obj-$(CONFIG_FS_VERITY) += enable.o \
init.o \
measure.o \
open.o \
+ pagecache.o \
read_metadata.o \
verify.o
diff --git a/fs/verity/enable.c b/fs/verity/enable.c
index 95ec42b84797..c9448074cce1 100644
--- a/fs/verity/enable.c
+++ b/fs/verity/enable.c
@@ -41,14 +41,15 @@ static int hash_one_block(const struct merkle_tree_params *params,
return 0;
}
-static int write_merkle_tree_block(struct inode *inode, const u8 *buf,
+static int write_merkle_tree_block(struct file *file, const u8 *buf,
unsigned long index,
const struct merkle_tree_params *params)
{
+ struct inode *inode = file_inode(file);
u64 pos = (u64)index << params->log_blocksize;
int err;
- err = inode->i_sb->s_vop->write_merkle_tree_block(inode, buf, pos,
+ err = inode->i_sb->s_vop->write_merkle_tree_block(file, buf, pos,
params->block_size);
if (err)
fsverity_err(inode, "Error %d writing Merkle tree block %lu",
@@ -135,7 +136,7 @@ static int build_merkle_tree(struct file *filp,
err = hash_one_block(params, &buffers[level]);
if (err)
goto out;
- err = write_merkle_tree_block(inode,
+ err = write_merkle_tree_block(filp,
buffers[level].data,
level_offset[level],
params);
@@ -155,7 +156,7 @@ static int build_merkle_tree(struct file *filp,
err = hash_one_block(params, &buffers[level]);
if (err)
goto out;
- err = write_merkle_tree_block(inode,
+ err = write_merkle_tree_block(filp,
buffers[level].data,
level_offset[level],
params);
@@ -265,8 +266,25 @@ static int enable_verity(struct file *filp,
}
/*
+ * Add the fsverity_info into the hash table before finishing the
+ * initialization so that we don't have to undo the enabling when memory
+ * allocation for the hash table fails. This is safe because looking up
+ * the fsverity_info always first checks the S_VERITY flag on the inode,
+ * which will only be set at the very end of the ->end_enable_verity
+ * method.
+ */
+ err = fsverity_set_info(vi);
+ if (err) {
+ fsverity_free_info(vi);
+ goto rollback;
+ }
+
+ /*
* Tell the filesystem to finish enabling verity on the file.
- * Serialized with ->begin_enable_verity() by the inode lock.
+ * Serialized with ->begin_enable_verity() by the inode lock. The file
+ * system needs to set the S_VERITY flag on the inode at the very end of
+ * the method, at which point the fsverity information can be accessed
+ * by other threads.
*/
inode_lock(inode);
err = vops->end_enable_verity(filp, desc, desc_size, params.tree_size);
@@ -274,19 +292,10 @@ static int enable_verity(struct file *filp,
if (err) {
fsverity_err(inode, "%ps() failed with err %d",
vops->end_enable_verity, err);
- fsverity_free_info(vi);
+ fsverity_remove_info(vi);
} else if (WARN_ON_ONCE(!IS_VERITY(inode))) {
+ fsverity_remove_info(vi);
err = -EINVAL;
- fsverity_free_info(vi);
- } else {
- /* Successfully enabled verity */
-
- /*
- * Readers can start using the inode's verity info immediately,
- * so it can't be rolled back once set. So don't set it until
- * just after the filesystem has successfully enabled verity.
- */
- fsverity_set_info(inode, vi);
}
out:
kfree(params.hashstate);
diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h
index dd20b138d452..2887cb849cec 100644
--- a/fs/verity/fsverity_private.h
+++ b/fs/verity/fsverity_private.h
@@ -11,6 +11,7 @@
#define pr_fmt(fmt) "fs-verity: " fmt
#include <linux/fsverity.h>
+#include <linux/rhashtable.h>
/*
* Implementation limit: maximum depth of the Merkle tree. For now 8 is plenty;
@@ -63,17 +64,18 @@ struct merkle_tree_params {
* fsverity_info - cached verity metadata for an inode
*
* When a verity file is first opened, an instance of this struct is allocated
- * and a pointer to it is stored in the file's in-memory inode. It remains
- * until the inode is evicted. It caches information about the Merkle tree
- * that's needed to efficiently verify data read from the file. It also caches
- * the file digest. The Merkle tree pages themselves are not cached here, but
- * the filesystem may cache them.
+ * and a pointer to it is stored in the global hash table, indexed by the inode
+ * pointer value. It remains alive until the inode is evicted. It caches
+ * information about the Merkle tree that's needed to efficiently verify data
+ * read from the file. It also caches the file digest. The Merkle tree pages
+ * themselves are not cached here, but the filesystem may cache them.
*/
struct fsverity_info {
+ struct rhash_head rhash_head;
struct merkle_tree_params tree_params;
u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE];
u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE];
- const struct inode *inode;
+ struct inode *inode;
unsigned long *hash_block_verified;
};
@@ -124,12 +126,12 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params,
unsigned int log_blocksize,
const u8 *salt, size_t salt_size);
-struct fsverity_info *fsverity_create_info(const struct inode *inode,
+struct fsverity_info *fsverity_create_info(struct inode *inode,
struct fsverity_descriptor *desc);
-void fsverity_set_info(struct inode *inode, struct fsverity_info *vi);
-
+int fsverity_set_info(struct fsverity_info *vi);
void fsverity_free_info(struct fsverity_info *vi);
+void fsverity_remove_info(struct fsverity_info *vi);
int fsverity_get_descriptor(struct inode *inode,
struct fsverity_descriptor **desc_ret);
diff --git a/fs/verity/open.c b/fs/verity/open.c
index 77b1c977af02..dfa0d1afe0fe 100644
--- a/fs/verity/open.c
+++ b/fs/verity/open.c
@@ -12,6 +12,14 @@
#include <linux/slab.h>
static struct kmem_cache *fsverity_info_cachep;
+static struct rhashtable fsverity_info_hash;
+
+static const struct rhashtable_params fsverity_info_hash_params = {
+ .key_len = sizeof_field(struct fsverity_info, inode),
+ .key_offset = offsetof(struct fsverity_info, inode),
+ .head_offset = offsetof(struct fsverity_info, rhash_head),
+ .automatic_shrinking = true,
+};
/**
* fsverity_init_merkle_tree_params() - initialize Merkle tree parameters
@@ -175,7 +183,7 @@ static void compute_file_digest(const struct fsverity_hash_alg *hash_alg,
* appended builtin signature), and check the signature if present. The
* fsverity_descriptor must have already undergone basic validation.
*/
-struct fsverity_info *fsverity_create_info(const struct inode *inode,
+struct fsverity_info *fsverity_create_info(struct inode *inode,
struct fsverity_descriptor *desc)
{
struct fsverity_info *vi;
@@ -241,33 +249,19 @@ fail:
return ERR_PTR(err);
}
-void fsverity_set_info(struct inode *inode, struct fsverity_info *vi)
+int fsverity_set_info(struct fsverity_info *vi)
{
- /*
- * Multiple tasks may race to set the inode's verity info pointer, so
- * use cmpxchg_release(). This pairs with the smp_load_acquire() in
- * fsverity_get_info(). I.e., publish the pointer with a RELEASE
- * barrier so that other tasks can ACQUIRE it.
- */
- if (cmpxchg_release(fsverity_info_addr(inode), NULL, vi) != NULL) {
- /* Lost the race, so free the verity info we allocated. */
- fsverity_free_info(vi);
- /*
- * Afterwards, the caller may access the inode's verity info
- * directly, so make sure to ACQUIRE the winning verity info.
- */
- (void)fsverity_get_info(inode);
- }
+ return rhashtable_lookup_insert_fast(&fsverity_info_hash,
+ &vi->rhash_head,
+ fsverity_info_hash_params);
}
-void fsverity_free_info(struct fsverity_info *vi)
+struct fsverity_info *__fsverity_get_info(const struct inode *inode)
{
- if (!vi)
- return;
- kfree(vi->tree_params.hashstate);
- kvfree(vi->hash_block_verified);
- kmem_cache_free(fsverity_info_cachep, vi);
+ return rhashtable_lookup_fast(&fsverity_info_hash, &inode,
+ fsverity_info_hash_params);
}
+EXPORT_SYMBOL_GPL(__fsverity_get_info);
static bool validate_fsverity_descriptor(struct inode *inode,
const struct fsverity_descriptor *desc,
@@ -352,7 +346,7 @@ int fsverity_get_descriptor(struct inode *inode,
static int ensure_verity_info(struct inode *inode)
{
- struct fsverity_info *vi = fsverity_get_info(inode);
+ struct fsverity_info *vi = fsverity_get_info(inode), *found;
struct fsverity_descriptor *desc;
int err;
@@ -369,8 +363,19 @@ static int ensure_verity_info(struct inode *inode)
goto out_free_desc;
}
- fsverity_set_info(inode, vi);
- err = 0;
+ /*
+ * Multiple tasks may race to set the inode's verity info, in which case
+ * we might find an existing fsverity_info in the hash table.
+ */
+ found = rhashtable_lookup_get_insert_fast(&fsverity_info_hash,
+ &vi->rhash_head,
+ fsverity_info_hash_params);
+ if (found) {
+ fsverity_free_info(vi);
+ if (IS_ERR(found))
+ err = PTR_ERR(found);
+ }
+
out_free_desc:
kfree(desc);
return err;
@@ -384,25 +389,32 @@ int __fsverity_file_open(struct inode *inode, struct file *filp)
}
EXPORT_SYMBOL_GPL(__fsverity_file_open);
-int __fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr)
+void fsverity_free_info(struct fsverity_info *vi)
{
- if (attr->ia_valid & ATTR_SIZE)
- return -EPERM;
- return 0;
+ kfree(vi->tree_params.hashstate);
+ kvfree(vi->hash_block_verified);
+ kmem_cache_free(fsverity_info_cachep, vi);
}
-EXPORT_SYMBOL_GPL(__fsverity_prepare_setattr);
-void __fsverity_cleanup_inode(struct inode *inode)
+void fsverity_remove_info(struct fsverity_info *vi)
{
- struct fsverity_info **vi_addr = fsverity_info_addr(inode);
+ rhashtable_remove_fast(&fsverity_info_hash, &vi->rhash_head,
+ fsverity_info_hash_params);
+ fsverity_free_info(vi);
+}
- fsverity_free_info(*vi_addr);
- *vi_addr = NULL;
+void fsverity_cleanup_inode(struct inode *inode)
+{
+ struct fsverity_info *vi = fsverity_get_info(inode);
+
+ if (vi)
+ fsverity_remove_info(vi);
}
-EXPORT_SYMBOL_GPL(__fsverity_cleanup_inode);
void __init fsverity_init_info_cache(void)
{
+ if (rhashtable_init(&fsverity_info_hash, &fsverity_info_hash_params))
+ panic("failed to initialize fsverity hash\n");
fsverity_info_cachep = KMEM_CACHE_USERCOPY(
fsverity_info,
SLAB_RECLAIM_ACCOUNT | SLAB_PANIC,
diff --git a/fs/verity/pagecache.c b/fs/verity/pagecache.c
new file mode 100644
index 000000000000..1819314ecaa3
--- /dev/null
+++ b/fs/verity/pagecache.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019 Google LLC
+ */
+
+#include <linux/export.h>
+#include <linux/fsverity.h>
+#include <linux/pagemap.h>
+
+/**
+ * generic_read_merkle_tree_page - generic ->read_merkle_tree_page helper
+ * @inode: inode containing the Merkle tree
+ * @index: 0-based index of the Merkle tree page in the inode
+ *
+ * The caller needs to adjust @index from the Merkle-tree relative index passed
+ * to ->read_merkle_tree_page to the actual index where the Merkle tree is
+ * stored in the page cache for @inode.
+ */
+struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index)
+{
+ struct folio *folio;
+
+ folio = read_mapping_folio(inode->i_mapping, index, NULL);
+ if (IS_ERR(folio))
+ return ERR_CAST(folio);
+ return folio_file_page(folio, index);
+}
+EXPORT_SYMBOL_GPL(generic_read_merkle_tree_page);
+
+/**
+ * generic_readahead_merkle_tree() - generic ->readahead_merkle_tree helper
+ * @inode: inode containing the Merkle tree
+ * @index: 0-based index of the first Merkle tree page to read ahead in the
+ * inode
+ * @nr_pages: the number of Merkle tree pages that should be read ahead
+ *
+ * The caller needs to adjust @index from the Merkle-tree relative index passed
+ * to ->read_merkle_tree_page to the actual index where the Merkle tree is
+ * stored in the page cache for @inode.
+ */
+void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index,
+ unsigned long nr_pages)
+{
+ struct folio *folio;
+
+ lockdep_assert_held(&inode->i_mapping->invalidate_lock);
+
+ folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0);
+ if (folio == ERR_PTR(-ENOENT) ||
+ (!IS_ERR(folio) && !folio_test_uptodate(folio))) {
+ DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
+
+ page_cache_ra_unbounded(&ractl, nr_pages, 0);
+ }
+ if (!IS_ERR(folio))
+ folio_put(folio);
+}
+EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree);
diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c
index cba5d6af4e04..b4c0892430cd 100644
--- a/fs/verity/read_metadata.c
+++ b/fs/verity/read_metadata.c
@@ -28,24 +28,33 @@ static int fsverity_read_merkle_tree(struct inode *inode,
if (offset >= end_offset)
return 0;
offs_in_page = offset_in_page(offset);
+ index = offset >> PAGE_SHIFT;
last_index = (end_offset - 1) >> PAGE_SHIFT;
/*
+ * Kick off readahead for the range we are going to read to ensure a
+ * single large sequential read instead of lots of small ones.
+ */
+ if (inode->i_sb->s_vop->readahead_merkle_tree) {
+ filemap_invalidate_lock_shared(inode->i_mapping);
+ inode->i_sb->s_vop->readahead_merkle_tree(
+ inode, index, last_index - index + 1);
+ filemap_invalidate_unlock_shared(inode->i_mapping);
+ }
+
+ /*
* Iterate through each Merkle tree page in the requested range and copy
* the requested portion to userspace. Note that the Merkle tree block
* size isn't important here, as we are returning a byte stream; i.e.,
* we can just work with pages even if the tree block size != PAGE_SIZE.
*/
- for (index = offset >> PAGE_SHIFT; index <= last_index; index++) {
- unsigned long num_ra_pages =
- min_t(unsigned long, last_index - index + 1,
- inode->i_sb->s_bdi->io_pages);
+ for (; index <= last_index; index++) {
unsigned int bytes_to_copy = min_t(u64, end_offset - offset,
PAGE_SIZE - offs_in_page);
struct page *page;
const void *virt;
- page = vops->read_merkle_tree_page(inode, index, num_ra_pages);
+ page = vops->read_merkle_tree_page(inode, index);
if (IS_ERR(page)) {
err = PTR_ERR(page);
fsverity_err(inode,
diff --git a/fs/verity/verify.c b/fs/verity/verify.c
index 86067c8b40cf..31797f9b24d0 100644
--- a/fs/verity/verify.c
+++ b/fs/verity/verify.c
@@ -19,9 +19,7 @@ struct fsverity_pending_block {
};
struct fsverity_verification_context {
- struct inode *inode;
struct fsverity_info *vi;
- unsigned long max_ra_pages;
/*
* This is the queue of data blocks that are pending verification. When
@@ -37,6 +35,50 @@ struct fsverity_verification_context {
static struct workqueue_struct *fsverity_read_workqueue;
+/**
+ * fsverity_readahead() - kick off readahead on fsverity hashes
+ * @vi: fsverity_info for the inode to be read
+ * @index: first file data page index that is being read
+ * @nr_pages: number of file data pages to be read
+ *
+ * Start readahead on the fsverity hashes that are needed to verify the file
+ * data in the range from @index to @index + @nr_pages (exclusive upper bound).
+ *
+ * To be called from the file systems' ->read_folio and ->readahead methods to
+ * ensure that the hashes are already cached on completion of the file data
+ * read if possible.
+ */
+void fsverity_readahead(struct fsverity_info *vi, pgoff_t index,
+ unsigned long nr_pages)
+{
+ struct inode *inode = vi->inode;
+ const struct merkle_tree_params *params = &vi->tree_params;
+ u64 start_hidx = (u64)index << params->log_blocks_per_page;
+ u64 end_hidx =
+ (((u64)index + nr_pages) << params->log_blocks_per_page) - 1;
+ int level;
+
+ if (!inode->i_sb->s_vop->readahead_merkle_tree)
+ return;
+
+ for (level = 0; level < params->num_levels; level++) {
+ unsigned long level_start = params->level_start[level];
+ unsigned long next_start_hidx = start_hidx >> params->log_arity;
+ unsigned long next_end_hidx = end_hidx >> params->log_arity;
+ pgoff_t start_idx = (level_start + next_start_hidx) >>
+ params->log_blocks_per_page;
+ pgoff_t end_idx = (level_start + next_end_hidx) >>
+ params->log_blocks_per_page;
+
+ inode->i_sb->s_vop->readahead_merkle_tree(
+ inode, start_idx, end_idx - start_idx + 1);
+
+ start_hidx = next_start_hidx;
+ end_hidx = next_end_hidx;
+ }
+}
+EXPORT_SYMBOL_GPL(fsverity_readahead);
+
/*
* Returns true if the hash block with index @hblock_idx in the tree, located in
* @hpage, has already been verified.
@@ -113,10 +155,10 @@ static bool is_hash_block_verified(struct fsverity_info *vi, struct page *hpage,
*
* Return: %true if the data block is valid, else %false.
*/
-static bool verify_data_block(struct inode *inode, struct fsverity_info *vi,
- const struct fsverity_pending_block *dblock,
- unsigned long max_ra_pages)
+static bool verify_data_block(struct fsverity_info *vi,
+ const struct fsverity_pending_block *dblock)
{
+ struct inode *inode = vi->inode;
const u64 data_pos = dblock->pos;
const struct merkle_tree_params *params = &vi->tree_params;
const unsigned int hsize = params->digest_size;
@@ -200,8 +242,7 @@ static bool verify_data_block(struct inode *inode, struct fsverity_info *vi,
(params->block_size - 1);
hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode,
- hpage_idx, level == 0 ? min(max_ra_pages,
- params->tree_pages - hpage_idx) : 0);
+ hpage_idx);
if (IS_ERR(hpage)) {
fsverity_err(inode,
"Error %ld reading Merkle tree page %lu",
@@ -272,14 +313,9 @@ error:
static void
fsverity_init_verification_context(struct fsverity_verification_context *ctx,
- struct inode *inode,
- unsigned long max_ra_pages)
+ struct fsverity_info *vi)
{
- struct fsverity_info *vi = *fsverity_info_addr(inode);
-
- ctx->inode = inode;
ctx->vi = vi;
- ctx->max_ra_pages = max_ra_pages;
ctx->num_pending = 0;
if (vi->tree_params.hash_alg->algo_id == HASH_ALGO_SHA256 &&
sha256_finup_2x_is_optimized())
@@ -322,8 +358,7 @@ fsverity_verify_pending_blocks(struct fsverity_verification_context *ctx)
}
for (i = 0; i < ctx->num_pending; i++) {
- if (!verify_data_block(ctx->inode, vi, &ctx->pending_blocks[i],
- ctx->max_ra_pages))
+ if (!verify_data_block(vi, &ctx->pending_blocks[i]))
return false;
}
fsverity_clear_pending_blocks(ctx);
@@ -359,6 +394,7 @@ static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx,
/**
* fsverity_verify_blocks() - verify data in a folio
+ * @vi: fsverity_info for the inode to be read
* @folio: the folio containing the data to verify
* @len: the length of the data to verify in the folio
* @offset: the offset of the data to verify in the folio
@@ -369,11 +405,12 @@ static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx,
*
* Return: %true if the data is valid, else %false.
*/
-bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset)
+bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio,
+ size_t len, size_t offset)
{
struct fsverity_verification_context ctx;
- fsverity_init_verification_context(&ctx, folio->mapping->host, 0);
+ fsverity_init_verification_context(&ctx, vi);
if (fsverity_add_data_blocks(&ctx, folio, len, offset) &&
fsverity_verify_pending_blocks(&ctx))
@@ -386,6 +423,7 @@ EXPORT_SYMBOL_GPL(fsverity_verify_blocks);
#ifdef CONFIG_BLOCK
/**
* fsverity_verify_bio() - verify a 'read' bio that has just completed
+ * @vi: fsverity_info for the inode to be read
* @bio: the bio to verify
*
* Verify the bio's data against the file's Merkle tree. All bio data segments
@@ -398,27 +436,12 @@ EXPORT_SYMBOL_GPL(fsverity_verify_blocks);
* filesystems) must instead call fsverity_verify_page() directly on each page.
* All filesystems must also call fsverity_verify_page() on holes.
*/
-void fsverity_verify_bio(struct bio *bio)
+void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio)
{
- struct inode *inode = bio_first_folio_all(bio)->mapping->host;
struct fsverity_verification_context ctx;
struct folio_iter fi;
- unsigned long max_ra_pages = 0;
-
- if (bio->bi_opf & REQ_RAHEAD) {
- /*
- * If this bio is for data readahead, then we also do readahead
- * of the first (largest) level of the Merkle tree. Namely,
- * when a Merkle tree page is read, we also try to piggy-back on
- * some additional pages -- up to 1/4 the number of data pages.
- *
- * This improves sequential read performance, as it greatly
- * reduces the number of I/O requests made to the Merkle tree.
- */
- max_ra_pages = bio->bi_iter.bi_size >> (PAGE_SHIFT + 2);
- }
- fsverity_init_verification_context(&ctx, inode, max_ra_pages);
+ fsverity_init_verification_context(&ctx, vi);
bio_for_each_folio_all(fi, bio) {
if (!fsverity_add_data_blocks(&ctx, fi.folio, fi.length,