Loading fs/dcache.c +36 −91 Original line number Original line Diff line number Diff line Loading @@ -2066,42 +2066,19 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, } } EXPORT_SYMBOL(d_add_ci); EXPORT_SYMBOL(d_add_ci); /* * Do the slow-case of the dentry name compare. * * Unlike the dentry_cmp() function, we need to atomically * load the name and length information, so that the * filesystem can rely on them, and can use the 'name' and * 'len' information without worrying about walking off the * end of memory etc. * * Thus the read_seqcount_retry() and the "duplicate" info * in arguments (the low-level filesystem should not look * at the dentry inode or name contents directly, since * rename can change them while we're in RCU mode). */ enum slow_d_compare { D_COMP_OK, D_COMP_NOMATCH, D_COMP_SEQRETRY, }; static noinline enum slow_d_compare slow_dentry_cmp( static inline bool d_same_name(const struct dentry *dentry, const struct dentry *parent, const struct dentry *parent, struct dentry *dentry, unsigned int seq, const struct qstr *name) const struct qstr *name) { { int tlen = dentry->d_name.len; if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) { const char *tname = dentry->d_name.name; if (dentry->d_name.len != name->len) return false; if (read_seqcount_retry(&dentry->d_seq, seq)) { return dentry_cmp(dentry, name->name, name->len) == 0; cpu_relax(); return D_COMP_SEQRETRY; } } if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) return parent->d_op->d_compare(parent, dentry, return D_COMP_NOMATCH; dentry->d_name.len, dentry->d_name.name, return D_COMP_OK; name) == 0; } } /** /** Loading Loading @@ -2180,6 +2157,9 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, * dentry compare, we will do seqretries until it is stable, * dentry compare, we will do seqretries until it is stable, * and if we end up with a successful lookup, we actually * and if we end up with a successful lookup, we actually * want to exit RCU lookup anyway. * want to exit RCU lookup anyway. * * Note that raw_seqcount_begin still *does* smp_rmb(), so * we are still guaranteed NUL-termination of ->d_name.name. */ */ seq = raw_seqcount_begin(&dentry->d_seq); seq = raw_seqcount_begin(&dentry->d_seq); if (dentry->d_parent != parent) if (dentry->d_parent != parent) Loading @@ -2188,23 +2168,27 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, continue; continue; if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { int tlen; const char *tname; if (dentry->d_name.hash != hashlen_hash(hashlen)) if (dentry->d_name.hash != hashlen_hash(hashlen)) continue; continue; *seqp = seq; tlen = dentry->d_name.len; switch (slow_dentry_cmp(parent, dentry, seq, name)) { tname = dentry->d_name.name; case D_COMP_OK: /* we want a consistent (name,len) pair */ return dentry; if (read_seqcount_retry(&dentry->d_seq, seq)) { case D_COMP_NOMATCH: cpu_relax(); continue; default: goto seqretry; goto seqretry; } } } if (parent->d_op->d_compare(parent, dentry, tlen, tname, name) != 0) continue; } else { if (dentry->d_name.hash_len != hashlen) if (dentry->d_name.hash_len != hashlen) continue; continue; if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) continue; } *seqp = seq; *seqp = seq; if (!dentry_cmp(dentry, str, hashlen_len(hashlen))) return dentry; return dentry; } } return NULL; return NULL; Loading Loading @@ -2253,9 +2237,7 @@ EXPORT_SYMBOL(d_lookup); */ */ struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) { { unsigned int len = name->len; unsigned int hash = name->hash; unsigned int hash = name->hash; const unsigned char *str = name->name; struct hlist_bl_head *b = d_hash(parent, hash); struct hlist_bl_head *b = d_hash(parent, hash); struct hlist_bl_node *node; struct hlist_bl_node *node; struct dentry *found = NULL; struct dentry *found = NULL; Loading Loading @@ -2294,21 +2276,8 @@ struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) if (d_unhashed(dentry)) if (d_unhashed(dentry)) goto next; goto next; /* if (!d_same_name(dentry, parent, name)) * It is safe to compare names since d_move() cannot * change the qstr (protected by d_lock). */ if (parent->d_flags & DCACHE_OP_COMPARE) { int tlen = dentry->d_name.len; const char *tname = dentry->d_name.name; if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) goto next; goto next; } else { if (dentry->d_name.len != len) goto next; if (dentry_cmp(dentry, str, len)) goto next; } dentry->d_lockref.count++; dentry->d_lockref.count++; found = dentry; found = dentry; Loading Loading @@ -2461,9 +2430,7 @@ struct dentry *d_alloc_parallel(struct dentry *parent, const struct qstr *name, const struct qstr *name, wait_queue_head_t *wq) wait_queue_head_t *wq) { { unsigned int len = name->len; unsigned int hash = name->hash; unsigned int hash = name->hash; const unsigned char *str = name->name; struct hlist_bl_head *b = in_lookup_hash(parent, hash); struct hlist_bl_head *b = in_lookup_hash(parent, hash); struct hlist_bl_node *node; struct hlist_bl_node *node; struct dentry *new = d_alloc(parent, name); struct dentry *new = d_alloc(parent, name); Loading Loading @@ -2514,17 +2481,8 @@ struct dentry *d_alloc_parallel(struct dentry *parent, continue; continue; if (dentry->d_parent != parent) if (dentry->d_parent != parent) continue; continue; if (parent->d_flags & DCACHE_OP_COMPARE) { if (!d_same_name(dentry, parent, name)) int tlen = dentry->d_name.len; const char *tname = dentry->d_name.name; if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) continue; continue; } else { if (dentry->d_name.len != len) continue; if (dentry_cmp(dentry, str, len)) continue; } hlist_bl_unlock(b); hlist_bl_unlock(b); /* now we can try to grab a reference */ /* now we can try to grab a reference */ if (!lockref_get_not_dead(&dentry->d_lockref)) { if (!lockref_get_not_dead(&dentry->d_lockref)) { Loading @@ -2551,17 +2509,8 @@ struct dentry *d_alloc_parallel(struct dentry *parent, goto mismatch; goto mismatch; if (unlikely(d_unhashed(dentry))) if (unlikely(d_unhashed(dentry))) goto mismatch; goto mismatch; if (parent->d_flags & DCACHE_OP_COMPARE) { if (unlikely(!d_same_name(dentry, parent, name))) int tlen = dentry->d_name.len; const char *tname = dentry->d_name.name; if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) goto mismatch; goto mismatch; } else { if (unlikely(dentry->d_name.len != len)) goto mismatch; if (unlikely(dentry_cmp(dentry, str, len))) goto mismatch; } /* OK, it *is* a hashed match; return it */ /* OK, it *is* a hashed match; return it */ spin_unlock(&dentry->d_lock); spin_unlock(&dentry->d_lock); dput(new); dput(new); Loading Loading @@ -2657,8 +2606,6 @@ EXPORT_SYMBOL(d_add); struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) { { struct dentry *alias; struct dentry *alias; int len = entry->d_name.len; const char *name = entry->d_name.name; unsigned int hash = entry->d_name.hash; unsigned int hash = entry->d_name.hash; spin_lock(&inode->i_lock); spin_lock(&inode->i_lock); Loading @@ -2672,9 +2619,7 @@ struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) continue; continue; if (alias->d_parent != entry->d_parent) if (alias->d_parent != entry->d_parent) continue; continue; if (alias->d_name.len != len) if (!d_same_name(alias, entry->d_parent, &entry->d_name)) continue; if (dentry_cmp(alias, name, len)) continue; continue; spin_lock(&alias->d_lock); spin_lock(&alias->d_lock); if (!d_unhashed(alias)) { if (!d_unhashed(alias)) { Loading Loading
fs/dcache.c +36 −91 Original line number Original line Diff line number Diff line Loading @@ -2066,42 +2066,19 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, } } EXPORT_SYMBOL(d_add_ci); EXPORT_SYMBOL(d_add_ci); /* * Do the slow-case of the dentry name compare. * * Unlike the dentry_cmp() function, we need to atomically * load the name and length information, so that the * filesystem can rely on them, and can use the 'name' and * 'len' information without worrying about walking off the * end of memory etc. * * Thus the read_seqcount_retry() and the "duplicate" info * in arguments (the low-level filesystem should not look * at the dentry inode or name contents directly, since * rename can change them while we're in RCU mode). */ enum slow_d_compare { D_COMP_OK, D_COMP_NOMATCH, D_COMP_SEQRETRY, }; static noinline enum slow_d_compare slow_dentry_cmp( static inline bool d_same_name(const struct dentry *dentry, const struct dentry *parent, const struct dentry *parent, struct dentry *dentry, unsigned int seq, const struct qstr *name) const struct qstr *name) { { int tlen = dentry->d_name.len; if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) { const char *tname = dentry->d_name.name; if (dentry->d_name.len != name->len) return false; if (read_seqcount_retry(&dentry->d_seq, seq)) { return dentry_cmp(dentry, name->name, name->len) == 0; cpu_relax(); return D_COMP_SEQRETRY; } } if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) return parent->d_op->d_compare(parent, dentry, return D_COMP_NOMATCH; dentry->d_name.len, dentry->d_name.name, return D_COMP_OK; name) == 0; } } /** /** Loading Loading @@ -2180,6 +2157,9 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, * dentry compare, we will do seqretries until it is stable, * dentry compare, we will do seqretries until it is stable, * and if we end up with a successful lookup, we actually * and if we end up with a successful lookup, we actually * want to exit RCU lookup anyway. * want to exit RCU lookup anyway. * * Note that raw_seqcount_begin still *does* smp_rmb(), so * we are still guaranteed NUL-termination of ->d_name.name. */ */ seq = raw_seqcount_begin(&dentry->d_seq); seq = raw_seqcount_begin(&dentry->d_seq); if (dentry->d_parent != parent) if (dentry->d_parent != parent) Loading @@ -2188,23 +2168,27 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, continue; continue; if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { int tlen; const char *tname; if (dentry->d_name.hash != hashlen_hash(hashlen)) if (dentry->d_name.hash != hashlen_hash(hashlen)) continue; continue; *seqp = seq; tlen = dentry->d_name.len; switch (slow_dentry_cmp(parent, dentry, seq, name)) { tname = dentry->d_name.name; case D_COMP_OK: /* we want a consistent (name,len) pair */ return dentry; if (read_seqcount_retry(&dentry->d_seq, seq)) { case D_COMP_NOMATCH: cpu_relax(); continue; default: goto seqretry; goto seqretry; } } } if (parent->d_op->d_compare(parent, dentry, tlen, tname, name) != 0) continue; } else { if (dentry->d_name.hash_len != hashlen) if (dentry->d_name.hash_len != hashlen) continue; continue; if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) continue; } *seqp = seq; *seqp = seq; if (!dentry_cmp(dentry, str, hashlen_len(hashlen))) return dentry; return dentry; } } return NULL; return NULL; Loading Loading @@ -2253,9 +2237,7 @@ EXPORT_SYMBOL(d_lookup); */ */ struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) { { unsigned int len = name->len; unsigned int hash = name->hash; unsigned int hash = name->hash; const unsigned char *str = name->name; struct hlist_bl_head *b = d_hash(parent, hash); struct hlist_bl_head *b = d_hash(parent, hash); struct hlist_bl_node *node; struct hlist_bl_node *node; struct dentry *found = NULL; struct dentry *found = NULL; Loading Loading @@ -2294,21 +2276,8 @@ struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) if (d_unhashed(dentry)) if (d_unhashed(dentry)) goto next; goto next; /* if (!d_same_name(dentry, parent, name)) * It is safe to compare names since d_move() cannot * change the qstr (protected by d_lock). */ if (parent->d_flags & DCACHE_OP_COMPARE) { int tlen = dentry->d_name.len; const char *tname = dentry->d_name.name; if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) goto next; goto next; } else { if (dentry->d_name.len != len) goto next; if (dentry_cmp(dentry, str, len)) goto next; } dentry->d_lockref.count++; dentry->d_lockref.count++; found = dentry; found = dentry; Loading Loading @@ -2461,9 +2430,7 @@ struct dentry *d_alloc_parallel(struct dentry *parent, const struct qstr *name, const struct qstr *name, wait_queue_head_t *wq) wait_queue_head_t *wq) { { unsigned int len = name->len; unsigned int hash = name->hash; unsigned int hash = name->hash; const unsigned char *str = name->name; struct hlist_bl_head *b = in_lookup_hash(parent, hash); struct hlist_bl_head *b = in_lookup_hash(parent, hash); struct hlist_bl_node *node; struct hlist_bl_node *node; struct dentry *new = d_alloc(parent, name); struct dentry *new = d_alloc(parent, name); Loading Loading @@ -2514,17 +2481,8 @@ struct dentry *d_alloc_parallel(struct dentry *parent, continue; continue; if (dentry->d_parent != parent) if (dentry->d_parent != parent) continue; continue; if (parent->d_flags & DCACHE_OP_COMPARE) { if (!d_same_name(dentry, parent, name)) int tlen = dentry->d_name.len; const char *tname = dentry->d_name.name; if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) continue; continue; } else { if (dentry->d_name.len != len) continue; if (dentry_cmp(dentry, str, len)) continue; } hlist_bl_unlock(b); hlist_bl_unlock(b); /* now we can try to grab a reference */ /* now we can try to grab a reference */ if (!lockref_get_not_dead(&dentry->d_lockref)) { if (!lockref_get_not_dead(&dentry->d_lockref)) { Loading @@ -2551,17 +2509,8 @@ struct dentry *d_alloc_parallel(struct dentry *parent, goto mismatch; goto mismatch; if (unlikely(d_unhashed(dentry))) if (unlikely(d_unhashed(dentry))) goto mismatch; goto mismatch; if (parent->d_flags & DCACHE_OP_COMPARE) { if (unlikely(!d_same_name(dentry, parent, name))) int tlen = dentry->d_name.len; const char *tname = dentry->d_name.name; if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) goto mismatch; goto mismatch; } else { if (unlikely(dentry->d_name.len != len)) goto mismatch; if (unlikely(dentry_cmp(dentry, str, len))) goto mismatch; } /* OK, it *is* a hashed match; return it */ /* OK, it *is* a hashed match; return it */ spin_unlock(&dentry->d_lock); spin_unlock(&dentry->d_lock); dput(new); dput(new); Loading Loading @@ -2657,8 +2606,6 @@ EXPORT_SYMBOL(d_add); struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) { { struct dentry *alias; struct dentry *alias; int len = entry->d_name.len; const char *name = entry->d_name.name; unsigned int hash = entry->d_name.hash; unsigned int hash = entry->d_name.hash; spin_lock(&inode->i_lock); spin_lock(&inode->i_lock); Loading @@ -2672,9 +2619,7 @@ struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) continue; continue; if (alias->d_parent != entry->d_parent) if (alias->d_parent != entry->d_parent) continue; continue; if (alias->d_name.len != len) if (!d_same_name(alias, entry->d_parent, &entry->d_name)) continue; if (dentry_cmp(alias, name, len)) continue; continue; spin_lock(&alias->d_lock); spin_lock(&alias->d_lock); if (!d_unhashed(alias)) { if (!d_unhashed(alias)) { Loading