The following should work:
--------------------------
struct S {
    int a;
    immutable(int)* p;
}

struct T {
    int a;
    shared(int)* p;
}

void main() {
    auto s = S();
    immutable(S)*p = &s; // cannot implicitly convert expression
                         // (& s) of type S* to immutable(S)*

    shared(S)* q = &s; // cannot implicitly convert expression
                       // (& s) of type S* to shared(S)*

    auto t = T();
    shared(T)* r = &t; // cannot implicitly convert expression
                       // (& t) of type T* to shared(T)*
}
---------------------------

It should also work not only with pointers, but all reference types (such as dynamic arrays).